Skip to content
Snippets Groups Projects
Commit 2bc7b800 authored by matt-kwong's avatar matt-kwong Committed by GitHub
Browse files

Merge pull request #8401 from matt-kwong/pull-request-params

Improve pull request test filtering
parents 247d413f 2c4453d4
No related branches found
No related tags found
No related merge requests found
...@@ -36,18 +36,16 @@ from subprocess import call, check_output ...@@ -36,18 +36,16 @@ from subprocess import call, check_output
class TestSuite: class TestSuite:
""" """
Contains tag to identify job as belonging to this test suite and Contains label to identify job as belonging to this test suite and
triggers to identify if changed files are relevant triggers to identify if changed files are relevant
""" """
def __init__(self, tags): def __init__(self, labels):
""" """
Build TestSuite to group tests by their tags Build TestSuite to group tests based on labeling
:param tag: string used to identify if a job belongs to this TestSuite :param label: strings that should match a jobs's platform, config, language, or test group
todo(mattkwong): Change the use of tag because do not want to depend on
job.shortname to identify what suite a test belongs to
""" """
self.triggers = [] self.triggers = []
self.tags = tags self.labels = labels
def add_trigger(self, trigger): def add_trigger(self, trigger):
""" """
...@@ -56,46 +54,75 @@ class TestSuite: ...@@ -56,46 +54,75 @@ class TestSuite:
""" """
self.triggers.append(trigger) self.triggers.append(trigger)
# Create test suites # Create test suites
_core_test_suite = TestSuite(['_c_']) _SANITY_TEST_SUITE = TestSuite(['sanity'])
_cpp_test_suite = TestSuite(['_c++_']) _CORE_TEST_SUITE = TestSuite(['c'])
_csharp_test_suite = TestSuite(['_csharp_']) _CPP_TEST_SUITE = TestSuite(['c++'])
_node_test_suite = TestSuite(['_node_']) _CSHARP_TEST_SUITE = TestSuite(['csharp'])
_objc_test_suite = TestSuite(['_objc_']) _NODE_TEST_SUITE = TestSuite(['node'])
_php_test_suite = TestSuite(['_php_', '_php7_']) _OBJC_TEST_SUITE = TestSuite(['objc'])
_python_test_suite = TestSuite(['_python_']) _PHP_TEST_SUITE = TestSuite(['php', 'php7'])
_ruby_test_suite = TestSuite(['_ruby']) _PYTHON_TEST_SUITE = TestSuite(['python'])
_all_test_suites = [_core_test_suite, _cpp_test_suite, _csharp_test_suite, _RUBY_TEST_SUITE = TestSuite(['ruby'])
_node_test_suite, _objc_test_suite, _php_test_suite, _LINUX_TEST_SUITE = TestSuite(['linux'])
_python_test_suite, _ruby_test_suite] _WINDOWS_TEST_SUITE = TestSuite(['windows'])
_MACOS_TEST_SUITE = TestSuite(['macos'])
_ALL_TEST_SUITES = [_SANITY_TEST_SUITE, _CORE_TEST_SUITE, _CPP_TEST_SUITE,
_CSHARP_TEST_SUITE, _NODE_TEST_SUITE, _OBJC_TEST_SUITE,
_PHP_TEST_SUITE, _PYTHON_TEST_SUITE, _RUBY_TEST_SUITE,
_LINUX_TEST_SUITE, _WINDOWS_TEST_SUITE, _MACOS_TEST_SUITE]
# Dictionary of whitelistable files where the key is a regex matching changed files # Dictionary of whitelistable files where the key is a regex matching changed files
# and the value is a list of tests that should be run. An empty list means that # and the value is a list of tests that should be run. An empty list means that
# the changed files should not trigger any tests. Any changed file that does not # the changed files should not trigger any tests. Any changed file that does not
# match any of these regexes will trigger all tests # match any of these regexes will trigger all tests
_WHITELIST_DICT = { _WHITELIST_DICT = {
'^templates/.*': [], '^doc/': [],
'^doc/.*': [], '^examples/': [],
'^examples/.*': [], '^include/grpc\+\+/': [_CPP_TEST_SUITE],
'^summerofcode/.*': [], '^summerofcode/': [],
'.*README.md$': [], '^src/cpp/': [_CPP_TEST_SUITE],
'.*LICENSE$': [], '^src/csharp/': [_CSHARP_TEST_SUITE],
'^src/cpp.*': [_cpp_test_suite], '^src/node/': [_NODE_TEST_SUITE],
'^src/csharp.*': [_csharp_test_suite], '^src/objective\-c/': [_OBJC_TEST_SUITE],
'^src/node.*': [_node_test_suite], '^src/php/': [_PHP_TEST_SUITE],
'^src/objective-c.*': [_objc_test_suite], '^src/python/': [_PYTHON_TEST_SUITE],
'^src/php.*': [_php_test_suite], '^src/ruby/': [_RUBY_TEST_SUITE],
'^src/python.*': [_python_test_suite], '^templates/': [_SANITY_TEST_SUITE],
'^src/ruby.*': [_ruby_test_suite], '^test/core/': [_CORE_TEST_SUITE],
'^test/core.*': [_core_test_suite], '^test/cpp/': [_CPP_TEST_SUITE],
'^test/cpp.*': [_cpp_test_suite], '^test/distrib/cpp/': [_CPP_TEST_SUITE],
'^test/distrib/cpp.*': [_cpp_test_suite], '^test/distrib/csharp/': [_CSHARP_TEST_SUITE],
'^test/distrib/csharp.*': [_csharp_test_suite], '^test/distrib/node/': [_NODE_TEST_SUITE],
'^test/distrib/node.*': [_node_test_suite], '^test/distrib/php/': [_PHP_TEST_SUITE],
'^test/distrib/php.*': [_php_test_suite], '^test/distrib/python/': [_PYTHON_TEST_SUITE],
'^test/distrib/python.*': [_python_test_suite], '^test/distrib/ruby/': [_RUBY_TEST_SUITE],
'^test/distrib/ruby.*': [_ruby_test_suite] '^vsprojects/': [_WINDOWS_TEST_SUITE],
'binding\.gyp$': [_NODE_TEST_SUITE],
'composer\.json$': [_PHP_TEST_SUITE],
'config\.m4$': [_PHP_TEST_SUITE],
'CONTRIBUTING\.md$': [],
'Gemfile$': [_RUBY_TEST_SUITE],
'grpc.def$': [_WINDOWS_TEST_SUITE],
'grpc\.gemspec$': [_RUBY_TEST_SUITE],
'gRPC\.podspec$': [_OBJC_TEST_SUITE],
'gRPC\-Core\.podspec$': [_OBJC_TEST_SUITE],
'gRPC\-ProtoRPC\.podspec$': [_OBJC_TEST_SUITE],
'gRPC\-RxLibrary\.podspec$': [_OBJC_TEST_SUITE],
'INSTALL\.md$': [],
'LICENSE$': [],
'MANIFEST\.md$': [],
'package\.json$': [_PHP_TEST_SUITE],
'package\.xml$': [_PHP_TEST_SUITE],
'PATENTS$': [],
'PYTHON\-MANIFEST\.in$': [_PYTHON_TEST_SUITE],
'README\.md$': [],
'requirements\.txt$': [_PYTHON_TEST_SUITE],
'setup\.cfg$': [_PYTHON_TEST_SUITE],
'setup\.py$': [_PYTHON_TEST_SUITE]
} }
# Add all triggers to their respective test suites # Add all triggers to their respective test suites
for trigger, test_suites in _WHITELIST_DICT.iteritems(): for trigger, test_suites in _WHITELIST_DICT.iteritems():
for test_suite in test_suites: for test_suite in test_suites:
...@@ -106,10 +133,6 @@ def _get_changed_files(base_branch): ...@@ -106,10 +133,6 @@ def _get_changed_files(base_branch):
""" """
Get list of changed files between current branch and base of target merge branch Get list of changed files between current branch and base of target merge branch
""" """
# git fetch might need to be called on Jenkins slave
# todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this
# call(['git', 'fetch'])
# Get file changes between branch and merge-base of specified branch # Get file changes between branch and merge-base of specified branch
# Not combined to be Windows friendly # Not combined to be Windows friendly
base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip()
...@@ -129,27 +152,17 @@ def _can_skip_tests(file_names, triggers): ...@@ -129,27 +152,17 @@ def _can_skip_tests(file_names, triggers):
return True return True
def _remove_irrelevant_tests(tests, tag): def _remove_irrelevant_tests(tests, skippable_labels):
""" """
Filters out tests by config or language - will not remove sanitizer tests Filters out tests by config or language - will not remove sanitizer tests
:param tests: list of all tests generated by run_tests_matrix.py :param tests: list of all tests generated by run_tests_matrix.py
:param tag: string representing language or config to filter - "_(language)_" or "_(config)" :param skippable_labels: list of languages and platforms with skippable tests
:return: list of relevant tests
"""
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname
return [test for test in tests if tag not in test.shortname or
any(san_tag in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])]
def _remove_sanitizer_tests(tests):
"""
Filters out sanitizer tests
:param tests: list of all tests generated by run_tests_matrix.py
:return: list of relevant tests :return: list of relevant tests
""" """
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname # test.labels[0] is platform and test.labels[2] is language
return [test for test in tests if # We skip a test if both are considered safe to skip
all(san_tag not in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] return [test for test in tests if test.labels[0] not in skippable_labels or \
test.labels[2] not in skippable_labels]
def filter_tests(tests, base_branch): def filter_tests(tests, base_branch):
...@@ -158,7 +171,7 @@ def filter_tests(tests, base_branch): ...@@ -158,7 +171,7 @@ def filter_tests(tests, base_branch):
:param tests: list of all tests generated by run_tests_matrix.py :param tests: list of all tests generated by run_tests_matrix.py
:return: list of relevant tests :return: list of relevant tests
""" """
print("Finding file differences between %s repo and current branch...\n" % base_branch) print("Finding file differences between gRPC %s branch and pull request...\n" % base_branch)
changed_files = _get_changed_files(base_branch) changed_files = _get_changed_files(base_branch)
for changed_file in changed_files: for changed_file in changed_files:
print(changed_file) print(changed_file)
...@@ -170,15 +183,13 @@ def filter_tests(tests, base_branch): ...@@ -170,15 +183,13 @@ def filter_tests(tests, base_branch):
for changed_file in changed_files: for changed_file in changed_files:
if not re.match(all_triggers, changed_file): if not re.match(all_triggers, changed_file):
return(tests) return(tests)
# Filter out tests by language # Figure out which language and platform tests to run
for test_suite in _all_test_suites: skippable_labels = []
for test_suite in _ALL_TEST_SUITES:
if _can_skip_tests(changed_files, test_suite.triggers): if _can_skip_tests(changed_files, test_suite.triggers):
for tag in test_suite.tags: for label in test_suite.labels:
print(" Filtering %s tests" % tag) print(" Filtering %s tests" % label)
tests = _remove_irrelevant_tests(tests, tag) skippable_labels.append(label)
# Sanitizer tests skipped if core and c++ are skipped
if _can_skip_tests(changed_files, _cpp_test_suite.triggers + _core_test_suite.triggers):
print(" Filtering Sanitizer tests")
tests = _remove_sanitizer_tests(tests)
tests = _remove_irrelevant_tests(tests, skippable_labels)
return tests return tests
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment