From b7ea4ab4d7360fd37d13e91fb0b1dd4ab407dde0 Mon Sep 17 00:00:00 2001
From: Yong Ni <yongni@google.com>
Date: Mon, 26 Jun 2017 15:24:40 -0700
Subject: [PATCH] Created test driver to run matrix test against different
docker images in GCR and generate a Junit like test result file.
---
tools/interop_matrix/README.md | 12 +-
tools/interop_matrix/create_matrix_images.py | 1 -
.../run_interop_matrix_tests.py | 189 ++++++++++++++++++
tools/run_tests/python_utils/report_utils.py | 26 ++-
4 files changed, 215 insertions(+), 13 deletions(-)
create mode 100755 tools/interop_matrix/run_interop_matrix_tests.py
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index 8493099d1a..14fe82b9a1 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -11,7 +11,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r
- `--git_checkout` enables git checkout grpc release branch/tag.
- `--release` specifies a git release tag. Make sure it is a valid tag in the grpc github rep.
- `--language` specifies a language.
- For examle, To build all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/create_matrix_images.py --git_checkout --release=all`.
+ For example, To build all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/create_matrix_images.py --git_checkout --release=all`.
- Verify the newly created docker images are uploaded to GCR. For example:
- `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos.
- `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_go1.7` show tags for a image repo.
@@ -30,11 +30,17 @@ from specific releases/tag, are used to test version compatiblity between gRPC r
- `LANG=go KEEP_IMAGE=1 ./create_testcases.sh` will generate `./testcases/go__master` and keep the local docker image so it can be invoked simply via `./testcases/go__master`. Note: remove local docker images manually afterwards with `docker rmi <image_id>`.
- Stage and commit the generated test case file `./testcases/<lang>__<release>`.
-## Instructions for running test cases against a GCR image
+## Instructions for running test cases against GCR images
+- Run `tools/interop_matrix/run_interop_matrix_tests.py`. Useful options:
+ - `--release` specifies a git release tag. Defaults to `--release=master`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
+ - `--language` specifies a language. Defaults to `--language=all`.
+ For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`.
+- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml').
+
+## Instructions for running test cases against a GCR image manually
- Run test cases by specifying `docker_image` variable inline with the test case script created above.
For example:
- `docker_image=gcr.io/grpc-testing/grpc_interop_go1.7:master ./testcases/go__master` will run go__master test cases against `go1.7` with gRPC release `master` docker image in GCR.
-
Note:
- File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`).
diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py
index 214ae0c3fd..119cbd2e88 100755
--- a/tools/interop_matrix/create_matrix_images.py
+++ b/tools/interop_matrix/create_matrix_images.py
@@ -253,5 +253,4 @@ for lang in languages:
# docker image name must be in the format <gcr_path>/<image>:<gcr_tag>
assert image.startswith(args.gcr_path) and image.find(':') != -1
- # subprocess.call(['gcloud', 'docker', '--', 'push', image])
subprocess.call(['gcloud', 'docker', '--', 'push', image])
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
new file mode 100755
index 0000000000..3273c3d759
--- /dev/null
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python2.7
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run tests using docker images in Google Container Registry per matrix."""
+
+from __future__ import print_function
+
+import argparse
+import atexit
+import json
+import multiprocessing
+import os
+import re
+import subprocess
+import sys
+import uuid
+
+# Langauage Runtime Matrix
+import client_matrix
+
+python_util_dir = os.path.abspath(os.path.join(
+ os.path.dirname(__file__), '../run_tests/python_utils'))
+sys.path.append(python_util_dir)
+import dockerjob
+import jobset
+import report_utils
+
+_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
+# All gRPC release tags, flattened, deduped and sorted.
+_RELEASES = sorted(list(set(
+ i for l in client_matrix.LANG_RELEASE_MATRIX.values() for i in l)))
+_TEST_TIMEOUT = 30
+
+argp = argparse.ArgumentParser(description='Run interop tests.')
+argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
+argp.add_argument('--gcr_path',
+ default='gcr.io/grpc-testing',
+ help='Path of docker images in Google Container Registry')
+
+argp.add_argument('--release',
+ default='master',
+ choices=['all', 'master'] + _RELEASES,
+ help='Release tags to test. When testing all '
+ 'releases defined in client_matrix.py, use "all".')
+
+argp.add_argument('-l', '--language',
+ choices=['all'] + sorted(_LANGUAGES),
+ nargs='+',
+ default=['all'],
+ help='Languages to test')
+
+argp.add_argument('--keep',
+ action='store_true',
+ help='keep the created local images after finishing the tests.')
+
+argp.add_argument('--report_file',
+ default='report.xml',
+ help='The result file to create.')
+
+args = argp.parse_args()
+
+
+def find_all_images_for_lang(lang):
+ """Find docker images for a language across releases and runtimes.
+
+ Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime.
+ """
+ # Find all defined releases.
+ if args.release == 'all':
+ releases = ['master'] + client_matrix.LANG_RELEASE_MATRIX[lang]
+ else:
+ # Look for a particular release.
+ if args.release not in ['master'] + client_matrix.LANG_RELEASE_MATRIX[lang]:
+ jobset.message('SKIPPED',
+ '%s for %s is not defined' % (args.release, lang),
+ do_newline=True)
+ return []
+ releases = [args.release]
+
+ # Images tuples keyed by runtime.
+ images = {}
+ for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
+ image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
+ output = subprocess.check_output(['gcloud', 'beta', 'container', 'images',
+ 'list-tags', '--format=json', image_path])
+ docker_image_list = json.loads(output)
+ # All images should have a single tag or no tag.
+ tags = [i['tags'][0] for i in docker_image_list if i['tags']]
+ jobset.message('START', 'Found images for %s: %s' % (image_path, tags),
+ do_newline=True)
+ skipped = len(docker_image_list) - len(tags)
+ jobset.message('START', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
+ do_newline=True)
+ # Filter tags based on the releases.
+ images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if
+ tag in releases]
+ return images
+
+# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime.
+_loaded_testcases = {}
+def find_test_cases(lang, release):
+ """Returns the list of test cases from testcase files per lang/release."""
+ file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
+ if not os.path.exists(file_tmpl % (lang, release)):
+ release = 'master'
+ testcases = file_tmpl % (lang, release)
+ if lang in _loaded_testcases.keys() and release in _loaded_testcases[lang].keys():
+ return _loaded_testcases[lang][release]
+
+ job_spec_list=[]
+ try:
+ with open(testcases) as f:
+ # Only line start with 'docker run' are test cases.
+ for line in f.readlines():
+ if line.startswith('docker run'):
+ m = re.search('--test_case=(.*)"', line)
+ shortname = m.group(1) if m else 'unknown_test'
+ spec = jobset.JobSpec(cmdline=line,
+ shortname=shortname,
+ timeout_seconds=_TEST_TIMEOUT,
+ shell=True)
+ job_spec_list.append(spec)
+ jobset.message('START',
+ 'Loaded %s tests from %s' % (len(job_spec_list), testcases),
+ do_newline=True)
+ except IOError as err:
+ jobset.message('FAILED', err, do_newline=True)
+ if lang not in _loaded_testcases.keys():
+ _loaded_testcases[lang] = {}
+ _loaded_testcases[lang][release]=job_spec_list
+ return job_spec_list
+
+_xml_report_tree = None
+def run_tests_for_lang(lang, runtime, images):
+ """Find and run all test cases for a language.
+
+ images is a list of (<release-tag>, <image-full-path>) tuple.
+ """
+ for image_tuple in images:
+ release, image = image_tuple
+ jobset.message('START', 'Testing %s' % image, do_newline=True)
+ _docker_images_cleanup.append(image)
+ job_spec_list = find_test_cases(lang,release)
+ num_failures, resultset = jobset.run(job_spec_list,
+ newline_on_success=True,
+ add_env={'docker_image':image},
+ maxjobs=args.jobs)
+ if num_failures:
+ jobset.message('FAILED', 'Some tests failed', do_newline=True)
+ else:
+ jobset.message('SUCCESS', 'All tests passed', do_newline=True)
+
+ # Required, otherwise _xml_report_tree will be shadowed by local (undefined)
+ # reference in the next line.
+ global _xml_report_tree
+ _xml_report_tree = report_utils.add_junit_xml_results(
+ resultset,
+ 'grpc_interop_matrix',
+ '%s__%s:%s'%(lang,runtime,release),
+ str(uuid.uuid4()),
+ _xml_report_tree)
+
+_docker_images_cleanup = []
+def cleanup():
+ if not args.keep:
+ for image in _docker_images_cleanup:
+ dockerjob.remove_image(image, skip_nonexistent=True)
+
+atexit.register(cleanup)
+
+languages = args.language if args.language != ['all'] else _LANGUAGES
+for lang in languages:
+ docker_images = find_all_images_for_lang(lang)
+ for runtime in sorted(docker_images.keys()):
+ run_tests_for_lang(lang, runtime, docker_images[runtime])
+
+report_utils.create_xml_report_file(args.report_file, _xml_report_tree)
diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py
index 4b4c50a03e..9e602102ff 100644
--- a/tools/run_tests/python_utils/report_utils.py
+++ b/tools/run_tests/python_utils/report_utils.py
@@ -46,8 +46,22 @@ def _filter_msg(msg, output_format):
def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
suite_name='tests'):
"""Generate JUnit-like XML report."""
- root = ET.Element('testsuites')
- testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package,
+ tree = add_junit_xml_results(resultset, suite_package, suite_name, '1')
+ create_xml_report_file(xml_report, tree)
+
+def create_xml_report_file(xml_report, tree):
+ """Generate JUnit-like report file from xml tree ."""
+ # ensure the report directory exists
+ report_dir = os.path.dirname(os.path.abspath(xml_report))
+ if not os.path.exists(report_dir):
+ os.makedirs(report_dir)
+ tree.write(xml_report, encoding='UTF-8')
+
+def add_junit_xml_results(resultset, suite_package, suite_name, id,
+ old_tree=None):
+ """Returns a JUnit-like XML report tree with added test results."""
+ root = ET.Element('testsuites') if not old_tree else old_tree.getroot()
+ testsuite = ET.SubElement(root, 'testsuite', id=id, package=suite_package,
name=suite_name)
failure_count = 0
error_count = 0
@@ -67,13 +81,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
ET.SubElement(xml_test, 'skipped', message='Skipped')
testsuite.set('failures', str(failure_count))
testsuite.set('errors', str(error_count))
- # ensure the report directory exists
- report_dir = os.path.dirname(os.path.abspath(xml_report))
- if not os.path.exists(report_dir):
- os.makedirs(report_dir)
- tree = ET.ElementTree(root)
- tree.write(xml_report, encoding='UTF-8')
-
+ return ET.ElementTree(root)
def render_interop_html_report(
client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
--
GitLab