Skip to content
Snippets Groups Projects
Commit b9e66cc5 authored by Adele Zhou's avatar Adele Zhou
Browse files

Added test cases for Cloud gateway.

parent 84b7e8cb
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
tools/run_tests/interop_html_report.template
+ 27
− 15
View file @ b9e66cc5
......@@ -64,17 +64,8 @@
% endif
</%def>
% if num_failures > 1:
<p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
% elif num_failures:
<p><h2><font color="red">${num_failures} test failed!</font></h2></p>
% else:
<p><h2><font color="green">All tests passed!</font></h2></p>
% endif
% if cloud_to_prod:
<%def name="display_cloud_to_prod_result(prod_server)">
## Each column header is the client language.
<h2>Cloud to Prod</h2>
<table style="width:100%" border="1">
<tr bgcolor="#00BFFF">
<th>Client languages &#9658;<br/>Test Cases &#9660;</th>
......@@ -87,15 +78,32 @@
% for client_lang in client_langs:
<%
if test_case in auth_test_cases:
shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
shortname = 'cloud_to_prod_auth:%s:%s:%s' % (
prod_server, client_lang, test_case)
else:
shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
shortname = 'cloud_to_prod:%s:%s:%s' % (
prod_server, client_lang, test_case)
%>
${fill_one_test_result(shortname, resultset)}
% endfor
</tr>
% endfor
</table>
</%def>
% if num_failures > 1:
<p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
% elif num_failures:
<p><h2><font color="red">${num_failures} test failed!</font></h2></p>
% else:
<p><h2><font color="green">All tests passed!</font></h2></p>
% endif
% if cloud_to_prod:
% for prod_server in prod_servers:
<h2>Cloud to ${prod_server}</h2>
${display_cloud_to_prod_result(prod_server)}
% endfor
% endif
% if http2_interop:
......@@ -108,7 +116,9 @@
<th>${server_lang}</th>
% endfor
% if cloud_to_prod:
<th>prod</th>
% for prod_server in prod_servers:
<th>${prod_server}</th>
% endfor
% endif
</tr>
% for test_case in http2_cases:
......@@ -122,8 +132,10 @@
${fill_one_http2_test_result(shortname, resultset)}
% endfor
% if cloud_to_prod:
<% shortname = 'cloud_to_prod:http2:%s' % test_case %>
${fill_one_http2_test_result(shortname, resultset)}
% for prod_server in prod_servers:
<% shortname = 'cloud_to_prod:%s:http2:%s' % (prod_server, test_case) %>
${fill_one_http2_test_result(shortname, resultset)}
% endfor
% endif
</tr>
% endfor
......
tools/run_tests/report_utils.py
+ 3
− 1
View file @ b9e66cc5
......@@ -77,7 +77,7 @@ def render_junit_xml_report(resultset, xml_report):
def render_interop_html_report(
client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
resultset, num_failures, cloud_to_prod, http2_interop):
resultset, num_failures, cloud_to_prod, prod_servers, http2_interop):
"""Generate HTML report for interop tests."""
template_file = 'tools/run_tests/interop_html_report.template'
try:
......@@ -94,6 +94,7 @@ def render_interop_html_report(
sorted_http2_cases = sorted(http2_cases)
sorted_client_langs = sorted(client_langs)
sorted_server_langs = sorted(server_langs)
sorted_prod_servers = sorted(prod_servers)
args = {'client_langs': sorted_client_langs,
'server_langs': sorted_server_langs,
......@@ -103,6 +104,7 @@ def render_interop_html_report(
'resultset': resultset,
'num_failures': num_failures,
'cloud_to_prod': cloud_to_prod,
'prod_servers': sorted_prod_servers,
'http2_interop': http2_interop}
html_report_out_dir = 'reports'
......
tools/run_tests/run_interop_tests.py
+ 59
− 29
View file @ b9e66cc5
......@@ -422,12 +422,13 @@ def _job_kill_handler(job):
time.sleep(2)
def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
def cloud_to_prod_jobspec(language, test_case, server_host_name,
server_host_detail, docker_image=None, auth=False):
"""Creates jobspec for cloud-to-prod interop test"""
container_name = None
cmdargs = [
'--server_host_override=grpc-test.sandbox.googleapis.com',
'--server_host=grpc-test.sandbox.googleapis.com',
'--server_host=%s' % server_host_detail[0],
'--server_host_override=%s' % server_host_detail[1],
'--server_port=443',
'--use_tls=true',
'--test_case=%s' % test_case]
......@@ -440,7 +441,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
cwd = language.client_cwd
if docker_image:
container_name = dockerjob.random_name('interop_client_%s' % language.safename)
container_name = dockerjob.random_name('interop_client_%s' %
language.safename)
cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=cwd,
......@@ -455,7 +457,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
cmdline=cmdline,
cwd=cwd,
environ=environ,
shortname='%s:%s:%s' % (suite_name, language, test_case),
shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language,
test_case),
timeout_seconds=90,
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
......@@ -491,7 +494,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
cwd=cwd,
environ=environ,
shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name,
test_case),
test_case),
timeout_seconds=90,
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
......@@ -572,6 +575,21 @@ def aggregate_http2_results(stdout):
'percent': 1.0 * passed / (passed + failed)
}
# A dictionary of prod servers to test.
# Format: server_name: (server_host, server_host_override, errors_allowed)
# TODO(adelez): implement logic for errors_allowed where if the indicated tests
# fail, they don't impact the overall test result.
prod_servers = {
'default': ('grpc-test.sandbox.googleapis.com',
'grpc-test.sandbox.googleapis.com', False),
'gateway_v2': ('grpc-test2.sandbox.googleapis.com',
'grpc-test2.sandbox.googleapis.com', True),
'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com',
False),
'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com',
True)
}
argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(_LANGUAGES),
......@@ -589,6 +607,12 @@ argp.add_argument('--cloud_to_prod_auth',
action='store_const',
const=True,
help='Run cloud_to_prod_auth tests.')
argp.add_argument('--prod_servers',
choices=prod_servers.keys(),
default=['default'],
nargs='+',
help=('The servers to run cloud_to_prod and '
'cloud_to_prod_auth tests against.'))
argp.add_argument('-s', '--server',
choices=['all'] + sorted(_SERVERS),
action='append',
......@@ -688,32 +712,37 @@ try:
server_jobs[lang] = job
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
jobs = []
if args.cloud_to_prod:
for language in languages:
for test_case in _TEST_CASES:
if not test_case in language.unimplemented_test_cases():
if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
test_job = cloud_to_prod_jobspec(language, test_case,
docker_image=docker_images.get(str(language)))
jobs.append(test_job)
if args.http2_interop:
for test_case in _HTTP2_TEST_CASES:
test_job = cloud_to_prod_jobspec(http2Interop, test_case,
docker_image=docker_images.get(str(http2Interop)))
jobs.append(test_job)
for server_host_name in args.prod_servers:
for language in languages:
for test_case in _TEST_CASES:
if not test_case in language.unimplemented_test_cases():
if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
test_job = cloud_to_prod_jobspec(
language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)))
jobs.append(test_job)
if args.http2_interop:
for test_case in _HTTP2_TEST_CASES:
test_job = cloud_to_prod_jobspec(
http2Interop, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(http2Interop)))
jobs.append(test_job)
if args.cloud_to_prod_auth:
for language in languages:
for test_case in _AUTH_TEST_CASES:
if not test_case in language.unimplemented_test_cases():
test_job = cloud_to_prod_jobspec(language, test_case,
docker_image=docker_images.get(str(language)),
auth=True)
jobs.append(test_job)
for server_host_name in args.prod_servers:
for language in languages:
for test_case in _AUTH_TEST_CASES:
if not test_case in language.unimplemented_test_cases():
test_job = cloud_to_prod_jobspec(
language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)), auth=True)
jobs.append(test_job)
for server in args.override_server:
server_name = server[0]
......@@ -773,7 +802,8 @@ try:
report_utils.render_interop_html_report(
set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
_HTTP2_TEST_CASES, resultset, num_failures,
args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop)
args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers,
args.http2_interop)
finally:
# Check if servers are still running.
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment