make python test suites run in parallel

src/python/grpcio/tests/_runner.py

-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -143,10 +143,17 @@ class Runner(object):
 
   def run(self, suite):
     """See setuptools' test_runner setup argument for information."""
+    # only run test cases with id starting with given prefix
+    testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER')
+    filtered_cases = []
+    for case in _loader.iterate_suite_cases(suite):
+      if not testcase_filter or case.id().startswith(testcase_filter):
+        filtered_cases.append(case)
+
     # Ensure that every test case has no collision with any other test case in
     # the augmented results.
     augmented_cases = [AugmentedCase(case, uuid.uuid4())
-                       for case in _loader.iterate_suite_cases(suite)]
+                       for case in filtered_cases]
     case_id_by_case = dict((augmented_case.case, augmented_case.id)
                            for augmented_case in augmented_cases)
     result_out = StringIO.StringIO()

src/python/grpcio/tests/tests.json

+[
+  "_base_interface_test.AsyncEasyTest", 
+  "_base_interface_test.AsyncPeasyTest", 
+  "_base_interface_test.SyncEasyTest", 
+  "_base_interface_test.SyncPeasyTest", 
+  "_beta_features_test.BetaFeaturesTest", 
+  "_beta_features_test.ContextManagementAndLifecycleTest", 
+  "_channel_test.ChannelTest", 
+  "_connectivity_channel_test.ChannelConnectivityTest", 
+  "_core_over_links_base_interface_test.AsyncEasyTest", 
+  "_core_over_links_base_interface_test.AsyncPeasyTest", 
+  "_core_over_links_base_interface_test.SyncEasyTest", 
+  "_core_over_links_base_interface_test.SyncPeasyTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_implementations_test.ChannelCredentialsTest", 
+  "_insecure_interop_test.InsecureInteropTest", 
+  "_intermediary_low_test.CancellationTest", 
+  "_intermediary_low_test.EchoTest", 
+  "_intermediary_low_test.ExpirationTest", 
+  "_intermediary_low_test.LonelyClientTest", 
+  "_later_test.LaterTest", 
+  "_logging_pool_test.LoggingPoolTest", 
+  "_lonely_invocation_link_test.LonelyInvocationLinkTest", 
+  "_low_test.HangingServerShutdown", 
+  "_low_test.InsecureServerInsecureClient", 
+  "_not_found_test.NotFoundTest", 
+  "_sanity_test.Sanity", 
+  "_secure_interop_test.SecureInteropTest", 
+  "_transmission_test.RoundTripTest", 
+  "_transmission_test.TransmissionTest", 
+  "_utilities_test.ChannelConnectivityTest", 
+  "beta_python_plugin_test.PythonPluginTest", 
+  "cygrpc_test.InsecureServerInsecureClient", 
+  "cygrpc_test.SecureServerSecureClient", 
+  "cygrpc_test.TypeSmokeTest"
+]
\ No newline at end of file

src/python/grpcio/tests/unit/_sanity/__init__.py

+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+

src/python/grpcio/tests/unit/_sanity/_sanity_test.py

+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import unittest
+
+import tests
+
+
+class Sanity(unittest.TestCase):
+
+  def testTestsJsonUpToDate(self):
+    """Autodiscovers all test suites and checks that tests.json is up to date"""
+    loader = tests.Loader()
+    loader.loadTestsFromNames(['tests'])
+    test_suite_names = [
+        test_case_class.id().rsplit('.', 1)[0]
+        for test_case_class in tests._loader.iterate_suite_cases(loader.suite)]
+    test_suite_names = sorted(set(test_suite_names))
+
+    with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+      tests_json = json.load(tests_json_file)
+    self.assertListEqual(test_suite_names, tests_json)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)

tools/run_tests/build_python.sh

@@ -45,3 +45,5 @@ export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
 tox --notest
 
 $ROOT/.tox/py27/bin/python $ROOT/setup.py build
+$ROOT/.tox/py27/bin/python $ROOT/setup.py build_py
+$ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test

tools/run_tests/run_python.sh

@@ -42,7 +42,12 @@ export LDFLAGS="-L$ROOT/libs/$CONFIG"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
 
-tox
+if [ "$CONFIG" = "gcov" ]
+then
+  tox
+else
+  $ROOT/.tox/py27/bin/python $ROOT/setup.py test
+fi
 
 mkdir -p $ROOT/reports
 rm -rf $ROOT/reports/python-coverage

tools/run_tests/run_tests.py

@@ -353,15 +353,27 @@ class PythonLanguage(object):
     _check_compiler(self.args.compiler, ['default'])
 
   def test_specs(self):
+    # load list of known test suites
+    with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+      tests_json = json.load(tests_json_file)
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
     environment['PYVER'] = '2.7'
-    return [self.config.job_spec(
-        ['tools/run_tests/run_python.sh'],
-        None,
-        environ=environment,
-        shortname='py.test',
-        timeout_seconds=15*60
-    )]
+    if self.config.build_config != 'gcov':
+      return [self.config.job_spec(
+          ['tools/run_tests/run_python.sh'],
+          None,
+          environ=dict(environment.items() +
+                       [('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
+          shortname='py.test.%s' % suite_name,
+          timeout_seconds=5*60)
+          for suite_name in tests_json]
+    else:
+      return [self.config.job_spec(['tools/run_tests/run_python.sh'],
+                                   None,
+                                   environ=environment,
+                                   shortname='py.test.coverage',
+                                   timeout_seconds=15*60)]
+
 
   def pre_build_steps(self):
     return []