diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py
index 1d67fe3033e2d0880fedb512cc96ff0dbeaaf80f..7d64222ba0b53931f57adf0869bc80ee6dd98000 100755
--- a/tools/run_tests/dockerjob.py
+++ b/tools/run_tests/dockerjob.py
@@ -101,7 +101,7 @@ class DockerJob:
def __init__(self, spec):
self._spec = spec
- self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}, xml_report=None)
+ self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={})
self._container_name = spec.container_name
def mapped_port(self, port):
diff --git a/tools/run_tests/generate_reports.py b/tools/run_tests/generate_reports.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ba47e9c2efebf4ce20cfc8f65801854239e32f3
--- /dev/null
+++ b/tools/run_tests/generate_reports.py
@@ -0,0 +1,172 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Generate XML and HTML test reports."""
+
+import os
+import xml.etree.cElementTree as ET
+
+
+def render_xml_report(resultset, xml_report):
+ """Generate JUnit-like XML report."""
+ root = ET.Element('testsuites')
+ testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc',
+ name='tests')
+ for shortname, results in resultset.iteritems():
+ for result in results:
+ xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
+ if result.elapsed_time:
+ xml_test.set('time', str(result.elapsed_time))
+ ET.SubElement(xml_test, 'system-out').text = result.message
+ if result.state == 'FAILED':
+ ET.SubElement(xml_test, 'failure', message='Failure')
+ elif result.state == 'TIMEOUT':
+ ET.SubElement(xml_test, 'error', message='Timeout')
+ tree = ET.ElementTree(root)
+ tree.write(xml_report, encoding='UTF-8')
+
+
+# TODO(adelez): Use mako template.
+def fill_one_test_result(shortname, resultset, html_str):
+ if shortname in resultset:
+ # Because interop tests does not have runs_per_test flag, each test is run
+ # once. So there should only be one element for each result.
+ result = resultset[shortname][0]
+ if result.state == 'PASSED':
+ html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
+ else:
+ tooltip = ''
+ if result.returncode > 0 or result.message:
+ if result.returncode > 0:
+ tooltip = 'returncode: %d ' % result.returncode
+ if result.message:
+ escaped_msg = result.message.replace('"', '"')
+ tooltip = '%smessage: %s' % (tooltip, escaped_msg)
+ if result.state == 'FAILED':
+ html_str = '%s<td bgcolor=\"red\">' % html_str
+ if tooltip:
+ html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+ 'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' %
+ (html_str, tooltip))
+ else:
+ html_str = '%sFAIL</td>\n' % html_str
+ elif result.state == 'TIMEOUT':
+ html_str = '%s<td bgcolor=\"yellow\">' % html_str
+ if tooltip:
+ html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+ 'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n'
+ % (html_str, tooltip))
+ else:
+ html_str = '%sTIMEOUT</td>\n' % html_str
+ else:
+ html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
+
+ return html_str
+
+
+def render_html_report(client_langs, server_langs, test_cases, auth_test_cases,
+ resultset, num_failures, cloud_to_prod):
+ """Generate html report."""
+ sorted_test_cases = sorted(test_cases)
+ sorted_auth_test_cases = sorted(auth_test_cases)
+ sorted_client_langs = sorted(client_langs)
+ sorted_server_langs = sorted(server_langs)
+ html_str = ('<!DOCTYPE html>\n'
+ '<html lang=\"en\">\n'
+ '<head><title>Interop Test Result</title></head>\n'
+ '<body>\n')
+ if num_failures > 1:
+ html_str = (
+ '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' %
+ (html_str, num_failures))
+ elif num_failures:
+ html_str = (
+ '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' %
+ (html_str, num_failures))
+ else:
+ html_str = (
+ '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' %
+ html_str)
+ if cloud_to_prod:
+ # Each column header is the client language.
+ html_str = ('%s<h2>Cloud to Prod</h2>\n'
+ '<table style=\"width:100%%\" border=\"1\">\n'
+ '<tr bgcolor=\"#00BFFF\">\n'
+ '<th>Client languages ►</th>\n') % html_str
+ for client_lang in sorted_client_langs:
+ html_str = '%s<th>%s\n' % (html_str, client_lang)
+ html_str = '%s</tr>\n' % html_str
+ for test_case in sorted_test_cases + sorted_auth_test_cases:
+ html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
+ for client_lang in sorted_client_langs:
+ if not test_case in sorted_auth_test_cases:
+ shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
+ else:
+ shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
+ html_str = fill_one_test_result(shortname, resultset, html_str)
+ html_str = '%s</tr>\n' % html_str
+ html_str = '%s</table>\n' % html_str
+ if server_langs:
+ for test_case in sorted_test_cases:
+ # Each column header is the client language.
+ html_str = ('%s<h2>%s</h2>\n'
+ '<table style=\"width:100%%\" border=\"1\">\n'
+ '<tr bgcolor=\"#00BFFF\">\n'
+ '<th>Client languages ►<br/>'
+ 'Server languages ▼</th>\n') % (html_str, test_case)
+ for client_lang in sorted_client_langs:
+ html_str = '%s<th>%s\n' % (html_str, client_lang)
+ html_str = '%s</tr>\n' % html_str
+ # Each row head is the server language.
+ for server_lang in sorted_server_langs:
+ html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
+ # Fill up the cells with test result.
+ for client_lang in sorted_client_langs:
+ shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
+ client_lang, server_lang, test_case)
+ html_str = fill_one_test_result(shortname, resultset, html_str)
+ html_str = '%s</tr>\n' % html_str
+ html_str = '%s</table>\n' % html_str
+
+ html_str = ('%s\n'
+ '<script>\n'
+ '$(document).ready(function(){'
+ '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'
+ '});\n'
+ '</script>\n'
+ '</body>\n'
+ '</html>') % html_str
+
+ # Write to reports/index.html as set up in Jenkins plugin.
+ html_report_dir = 'reports'
+ if not os.path.exists(html_report_dir):
+ os.mkdir(html_report_dir)
+ html_file_path = os.path.join(html_report_dir, 'index.html')
+ with open(html_file_path, 'w') as f:
+ f.write(html_str)
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index a8ff9f613fbf55daf32e8914ae8128677262b4c3..95bd7c7256aadfd220af837fcd16c5ecebd5ef1d 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -39,7 +39,6 @@ import subprocess
import sys
import tempfile
import time
-import xml.etree.cElementTree as ET
_DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
@@ -190,14 +189,12 @@ class JobResult(object):
class Job(object):
"""Manages one job."""
- def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report):
+ def __init__(self, spec, bin_hash, newline_on_success, travis, add_env):
self._spec = spec
self._bin_hash = bin_hash
self._newline_on_success = newline_on_success
self._travis = travis
self._add_env = add_env.copy()
- self._xml_test = ET.SubElement(xml_report, 'testcase',
- name=self._spec.shortname) if xml_report is not None else None
self._retries = 0
self._timeout_retries = 0
self._suppress_failure_message = False
@@ -229,15 +226,12 @@ class Job(object):
self._tempfile.seek(0)
stdout = self._tempfile.read()
filtered_stdout = _filter_stdout(stdout)
- # TODO: looks like jenkins master is slow because parsing the junit results XMLs is not
- # implemented efficiently. This is an experiment to workaround the issue by making sure
- # results.xml file is small enough.
+ # TODO: looks like jenkins master is slow because parsing the junit
+ # results XMLs is not implemented efficiently. This is an experiment to
+ # workaround the issue by making sure results.xml file is small enough.
filtered_stdout = filtered_stdout[-128:]
self.result.message = filtered_stdout
self.result.elapsed_time = elapsed
- if self._xml_test is not None:
- self._xml_test.set('time', str(elapsed))
- ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
if self._process.returncode != 0:
if self._retries < self._spec.flake_retries:
message('FLAKE', '%s [ret=%d, pid=%d]' % (
@@ -256,8 +250,6 @@ class Job(object):
self.result.state = 'FAILED'
self.result.num_failures += 1
self.result.returncode = self._process.returncode
- if self._xml_test is not None:
- ET.SubElement(self._xml_test, 'failure', message='Failure')
else:
self._state = _SUCCESS
message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
@@ -285,9 +277,6 @@ class Job(object):
self.kill()
self.result.state = 'TIMEOUT'
self.result.num_failures += 1
- if self._xml_test is not None:
- ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
- ET.SubElement(self._xml_test, 'error', message='Timeout')
return self._state
def kill(self):
@@ -305,7 +294,7 @@ class Jobset(object):
"""Manages one run of jobs."""
def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
- stop_on_failure, add_env, cache, xml_report):
+ stop_on_failure, add_env, cache):
self._running = set()
self._check_cancelled = check_cancelled
self._cancelled = False
@@ -317,7 +306,6 @@ class Jobset(object):
self._cache = cache
self._stop_on_failure = stop_on_failure
self._hashes = {}
- self._xml_report = xml_report
self._add_env = add_env
self.resultset = {}
@@ -349,8 +337,7 @@ class Jobset(object):
bin_hash,
self._newline_on_success,
self._travis,
- self._add_env,
- self._xml_report)
+ self._add_env)
self._running.add(job)
self.resultset[job.GetSpec().shortname] = []
return True
@@ -424,13 +411,11 @@ def run(cmdlines,
infinite_runs=False,
stop_on_failure=False,
cache=None,
- xml_report=None,
add_env={}):
js = Jobset(check_cancelled,
maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
newline_on_success, travis, stop_on_failure, add_env,
- cache if cache is not None else NoCache(),
- xml_report)
+ cache if cache is not None else NoCache())
for cmdline in cmdlines:
if not js.start(cmdline):
break
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 729f962bb18f3a389a5eeff3f8904e57ed2f2cd4..efbfe1f269a2281fbd5aa6bd6934fdfc0ac1d5c7 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -33,7 +33,7 @@
import argparse
import dockerjob
import itertools
-import xml.etree.cElementTree as ET
+import generate_reports
import jobset
import multiprocessing
import os
@@ -471,126 +471,6 @@ def build_interop_image_jobspec(language, tag=None):
return build_job
-# TODO(adelez): Use mako template.
-def fill_one_test_result(shortname, resultset, html_str):
- if shortname in resultset:
- # Because interop tests does not have runs_per_test flag, each test is run
- # once. So there should only be one element for each result.
- result = resultset[shortname][0]
- if result.state == 'PASSED':
- html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
- else:
- tooltip = ''
- if result.returncode > 0 or result.message:
- if result.returncode > 0:
- tooltip = 'returncode: %d ' % result.returncode
- if result.message:
- escaped_msg = result.message.replace('"', '"')
- tooltip = '%smessage: %s' % (tooltip, escaped_msg)
- if result.state == 'FAILED':
- html_str = '%s<td bgcolor=\"red\">' % html_str
- if tooltip:
- html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
- 'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' %
- (html_str, tooltip))
- else:
- html_str = '%sFAIL</td>\n' % html_str
- elif result.state == 'TIMEOUT':
- html_str = '%s<td bgcolor=\"yellow\">' % html_str
- if tooltip:
- html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
- 'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n'
- % (html_str, tooltip))
- else:
- html_str = '%sTIMEOUT</td>\n' % html_str
- else:
- html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
-
- return html_str
-
-
-def render_html_report(client_langs, server_langs, resultset,
- num_failures):
- """Generate html report."""
- sorted_test_cases = sorted(_TEST_CASES)
- sorted_auth_test_cases = sorted(_AUTH_TEST_CASES)
- sorted_client_langs = sorted(client_langs)
- sorted_server_langs = sorted(server_langs)
- html_str = ('<!DOCTYPE html>\n'
- '<html lang=\"en\">\n'
- '<head><title>Interop Test Result</title></head>\n'
- '<body>\n')
- if num_failures > 1:
- html_str = (
- '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' %
- (html_str, num_failures))
- elif num_failures:
- html_str = (
- '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' %
- (html_str, num_failures))
- else:
- html_str = (
- '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' %
- html_str)
- if args.cloud_to_prod_auth or args.cloud_to_prod:
- # Each column header is the client language.
- html_str = ('%s<h2>Cloud to Prod</h2>\n'
- '<table style=\"width:100%%\" border=\"1\">\n'
- '<tr bgcolor=\"#00BFFF\">\n'
- '<th>Client languages ►</th>\n') % html_str
- for client_lang in sorted_client_langs:
- html_str = '%s<th>%s\n' % (html_str, client_lang)
- html_str = '%s</tr>\n' % html_str
- for test_case in sorted_test_cases + sorted_auth_test_cases:
- html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
- for client_lang in sorted_client_langs:
- if not test_case in sorted_auth_test_cases:
- shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
- else:
- shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
- html_str = fill_one_test_result(shortname, resultset, html_str)
- html_str = '%s</tr>\n' % html_str
- html_str = '%s</table>\n' % html_str
- if servers:
- for test_case in sorted_test_cases:
- # Each column header is the client language.
- html_str = ('%s<h2>%s</h2>\n'
- '<table style=\"width:100%%\" border=\"1\">\n'
- '<tr bgcolor=\"#00BFFF\">\n'
- '<th>Client languages ►<br/>'
- 'Server languages ▼</th>\n') % (html_str, test_case)
- for client_lang in sorted_client_langs:
- html_str = '%s<th>%s\n' % (html_str, client_lang)
- html_str = '%s</tr>\n' % html_str
- # Each row head is the server language.
- for server_lang in sorted_server_langs:
- html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
- # Fill up the cells with test result.
- for client_lang in sorted_client_langs:
- shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
- client_lang, server_lang, test_case)
- html_str = fill_one_test_result(shortname, resultset, html_str)
- html_str = '%s</tr>\n' % html_str
- html_str = '%s</table>\n' % html_str
-
- html_str = ('%s\n'
- '<script>\n'
- '$(document).ready(function(){'
- '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'
- '});\n'
- '</script>\n'
- '</body>\n'
- '</html>') % html_str
-
- # Write to reports/index.html as set up in Jenkins plugin.
- html_report_dir = 'reports'
- if not os.path.exists(html_report_dir):
- os.mkdir(html_report_dir)
- html_file_path = os.path.join(html_report_dir, 'index.html')
- with open(html_file_path, 'w') as f:
- f.write(html_str)
-
-
argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(_LANGUAGES),
@@ -740,22 +620,20 @@ try:
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
- root = ET.Element('testsuites')
- testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests')
-
num_failures, resultset = jobset.run(jobs, newline_on_success=True,
- maxjobs=args.jobs, xml_report=testsuite)
+ maxjobs=args.jobs)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
- tree = ET.ElementTree(root)
- tree.write('report.xml', encoding='UTF-8')
+ # Generate XML report.
+ generate_reports.render_xml_report(resultset, 'report.xml')
# Generate HTML report.
- render_html_report(set([str(l) for l in languages]), servers,
- resultset, num_failures)
+ generate_reports.render_html_report(
+ set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
+ resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod)
finally:
# Check if servers are still running.
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 4232637c7f83117b3edfe2f7e4e7004ee1155ae4..725b48123990944d96d5a598819c2c8d88229391 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -46,9 +46,9 @@ import sys
import tempfile
import traceback
import time
-import xml.etree.cElementTree as ET
import urllib2
+import generate_reports
import jobset
import watch_dirs
@@ -867,15 +867,11 @@ def _build_and_run(
else itertools.repeat(massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
- root = ET.Element('testsuites') if xml_report else None
- testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None
-
number_failures, resultset = jobset.run(
- all_runs, check_cancelled, newline_on_success=newline_on_success,
+ all_runs, check_cancelled, newline_on_success=newline_on_success,
travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache if not xml_report else None,
- xml_report=testsuite,
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
if resultset:
for k, v in resultset.iteritems():
@@ -894,8 +890,7 @@ def _build_and_run(
for antagonist in antagonists:
antagonist.kill()
if xml_report:
- tree = ET.ElementTree(root)
- tree.write(xml_report, encoding='UTF-8')
+ generate_reports.render_xml_report(resultset, xml_report)
number_failures, _ = jobset.run(
post_tests_steps, maxjobs=1, stop_on_failure=True,