From b9e66cc551b1480e363667bac1d2162f7da7f27d Mon Sep 17 00:00:00 2001
From: Adele Zhou <adelez@google.com>
Date: Wed, 3 Feb 2016 13:12:23 -0800
Subject: [PATCH] Added test cases for Cloud gateway.
---
tools/run_tests/interop_html_report.template | 42 ++++++----
tools/run_tests/report_utils.py | 4 +-
tools/run_tests/run_interop_tests.py | 88 +++++++++++++-------
3 files changed, 89 insertions(+), 45 deletions(-)
diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop_html_report.template
index 114a508c31..46cce426b7 100644
--- a/tools/run_tests/interop_html_report.template
+++ b/tools/run_tests/interop_html_report.template
@@ -64,17 +64,8 @@
% endif
</%def>
-% if num_failures > 1:
- <p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
-% elif num_failures:
- <p><h2><font color="red">${num_failures} test failed!</font></h2></p>
-% else:
- <p><h2><font color="green">All tests passed!</font></h2></p>
-% endif
-
-% if cloud_to_prod:
+<%def name="display_cloud_to_prod_result(prod_server)">
## Each column header is the client language.
- <h2>Cloud to Prod</h2>
<table style="width:100%" border="1">
<tr bgcolor="#00BFFF">
<th>Client languages ►<br/>Test Cases ▼</th>
@@ -87,15 +78,32 @@
% for client_lang in client_langs:
<%
if test_case in auth_test_cases:
- shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
+ shortname = 'cloud_to_prod_auth:%s:%s:%s' % (
+ prod_server, client_lang, test_case)
else:
- shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
+ shortname = 'cloud_to_prod:%s:%s:%s' % (
+ prod_server, client_lang, test_case)
%>
${fill_one_test_result(shortname, resultset)}
% endfor
</tr>
% endfor
</table>
+</%def>
+
+% if num_failures > 1:
+ <p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
+% elif num_failures:
+ <p><h2><font color="red">${num_failures} test failed!</font></h2></p>
+% else:
+ <p><h2><font color="green">All tests passed!</font></h2></p>
+% endif
+
+% if cloud_to_prod:
+ % for prod_server in prod_servers:
+ <h2>Cloud to ${prod_server}</h2>
+ ${display_cloud_to_prod_result(prod_server)}
+ % endfor
% endif
% if http2_interop:
@@ -108,7 +116,9 @@
<th>${server_lang}</th>
% endfor
% if cloud_to_prod:
- <th>prod</th>
+ % for prod_server in prod_servers:
+ <th>${prod_server}</th>
+ % endfor
% endif
</tr>
% for test_case in http2_cases:
@@ -122,8 +132,10 @@
${fill_one_http2_test_result(shortname, resultset)}
% endfor
% if cloud_to_prod:
- <% shortname = 'cloud_to_prod:http2:%s' % test_case %>
- ${fill_one_http2_test_result(shortname, resultset)}
+ % for prod_server in prod_servers:
+ <% shortname = 'cloud_to_prod:%s:http2:%s' % (prod_server, test_case) %>
+ ${fill_one_http2_test_result(shortname, resultset)}
+ % endfor
% endif
</tr>
% endfor
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 35f2069bee..437c4693a5 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -77,7 +77,7 @@ def render_junit_xml_report(resultset, xml_report):
def render_interop_html_report(
client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
- resultset, num_failures, cloud_to_prod, http2_interop):
+ resultset, num_failures, cloud_to_prod, prod_servers, http2_interop):
"""Generate HTML report for interop tests."""
template_file = 'tools/run_tests/interop_html_report.template'
try:
@@ -94,6 +94,7 @@ def render_interop_html_report(
sorted_http2_cases = sorted(http2_cases)
sorted_client_langs = sorted(client_langs)
sorted_server_langs = sorted(server_langs)
+ sorted_prod_servers = sorted(prod_servers)
args = {'client_langs': sorted_client_langs,
'server_langs': sorted_server_langs,
@@ -103,6 +104,7 @@ def render_interop_html_report(
'resultset': resultset,
'num_failures': num_failures,
'cloud_to_prod': cloud_to_prod,
+ 'prod_servers': sorted_prod_servers,
'http2_interop': http2_interop}
html_report_out_dir = 'reports'
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index fbc3b3cdf6..4a670e0c9d 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -422,12 +422,13 @@ def _job_kill_handler(job):
time.sleep(2)
-def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
+def cloud_to_prod_jobspec(language, test_case, server_host_name,
+ server_host_detail, docker_image=None, auth=False):
"""Creates jobspec for cloud-to-prod interop test"""
container_name = None
cmdargs = [
- '--server_host_override=grpc-test.sandbox.googleapis.com',
- '--server_host=grpc-test.sandbox.googleapis.com',
+ '--server_host=%s' % server_host_detail[0],
+ '--server_host_override=%s' % server_host_detail[1],
'--server_port=443',
'--use_tls=true',
'--test_case=%s' % test_case]
@@ -440,7 +441,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
cwd = language.client_cwd
if docker_image:
- container_name = dockerjob.random_name('interop_client_%s' % language.safename)
+ container_name = dockerjob.random_name('interop_client_%s' %
+ language.safename)
cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=cwd,
@@ -455,7 +457,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
cmdline=cmdline,
cwd=cwd,
environ=environ,
- shortname='%s:%s:%s' % (suite_name, language, test_case),
+ shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language,
+ test_case),
timeout_seconds=90,
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
@@ -491,7 +494,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
cwd=cwd,
environ=environ,
shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name,
- test_case),
+ test_case),
timeout_seconds=90,
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
@@ -572,6 +575,21 @@ def aggregate_http2_results(stdout):
'percent': 1.0 * passed / (passed + failed)
}
+# A dictionary of prod servers to test.
+# Format: server_name: (server_host, server_host_override, errors_allowed)
+# TODO(adelez): implement logic for errors_allowed where if the indicated tests
+# fail, they don't impact the overall test result.
+prod_servers = {
+ 'default': ('grpc-test.sandbox.googleapis.com',
+ 'grpc-test.sandbox.googleapis.com', False),
+ 'gateway_v2': ('grpc-test2.sandbox.googleapis.com',
+ 'grpc-test2.sandbox.googleapis.com', True),
+ 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com',
+ False),
+ 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com',
+ True)
+}
+
argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(_LANGUAGES),
@@ -589,6 +607,12 @@ argp.add_argument('--cloud_to_prod_auth',
action='store_const',
const=True,
help='Run cloud_to_prod_auth tests.')
+argp.add_argument('--prod_servers',
+ choices=prod_servers.keys(),
+ default=['default'],
+ nargs='+',
+ help=('The servers to run cloud_to_prod and '
+ 'cloud_to_prod_auth tests against.'))
argp.add_argument('-s', '--server',
choices=['all'] + sorted(_SERVERS),
action='append',
@@ -688,32 +712,37 @@ try:
server_jobs[lang] = job
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
-
jobs = []
if args.cloud_to_prod:
- for language in languages:
- for test_case in _TEST_CASES:
- if not test_case in language.unimplemented_test_cases():
- if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
- test_job = cloud_to_prod_jobspec(language, test_case,
- docker_image=docker_images.get(str(language)))
- jobs.append(test_job)
-
- if args.http2_interop:
- for test_case in _HTTP2_TEST_CASES:
- test_job = cloud_to_prod_jobspec(http2Interop, test_case,
- docker_image=docker_images.get(str(http2Interop)))
- jobs.append(test_job)
-
+ for server_host_name in args.prod_servers:
+ for language in languages:
+ for test_case in _TEST_CASES:
+ if not test_case in language.unimplemented_test_cases():
+ if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
+ test_job = cloud_to_prod_jobspec(
+ language, test_case, server_host_name,
+ prod_servers[server_host_name],
+ docker_image=docker_images.get(str(language)))
+ jobs.append(test_job)
+
+ if args.http2_interop:
+ for test_case in _HTTP2_TEST_CASES:
+ test_job = cloud_to_prod_jobspec(
+ http2Interop, test_case, server_host_name,
+ prod_servers[server_host_name],
+ docker_image=docker_images.get(str(http2Interop)))
+ jobs.append(test_job)
if args.cloud_to_prod_auth:
- for language in languages:
- for test_case in _AUTH_TEST_CASES:
- if not test_case in language.unimplemented_test_cases():
- test_job = cloud_to_prod_jobspec(language, test_case,
- docker_image=docker_images.get(str(language)),
- auth=True)
- jobs.append(test_job)
+ for server_host_name in args.prod_servers:
+ for language in languages:
+ for test_case in _AUTH_TEST_CASES:
+ if not test_case in language.unimplemented_test_cases():
+ test_job = cloud_to_prod_jobspec(
+ language, test_case, server_host_name,
+ prod_servers[server_host_name],
+ docker_image=docker_images.get(str(language)), auth=True)
+ jobs.append(test_job)
for server in args.override_server:
server_name = server[0]
@@ -773,7 +802,8 @@ try:
report_utils.render_interop_html_report(
set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
_HTTP2_TEST_CASES, resultset, num_failures,
- args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop)
+ args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers,
+ args.http2_interop)
finally:
# Check if servers are still running.
--
GitLab