run_tests.py

  def __str__(self):
    return 'sanity'

class NodeExpressLanguage(object):
  """Dummy Node express test target to enable running express performance
  benchmarks"""

  def __init__(self):
    self.platform = platform_string()

  def configure(self, config, args):
    self.config = config
    self.args = args
    _check_compiler(self.args.compiler, ['default', 'node0.12',
                                         'node4', 'node5', 'node6'])
    if self.args.compiler == 'default':
      self.node_version = '4'
    else:
      # Take off the word "node"
      self.node_version = self.args.compiler[4:]

  def test_specs(self):
    return []

  def pre_build_steps(self):
    if self.platform == 'windows':
      return [['tools\\run_tests\\helper_scripts\\pre_build_node.bat']]
    else:
      return [['tools/run_tests/helper_scripts/pre_build_node.sh', self.node_version]]

  def make_targets(self):
    return []

  def make_options(self):
    return []

  def build_steps(self):
    return []

  def post_tests_steps(self):
    return []

  def makefile_name(self):
    return 'Makefile'

  def dockerfile_dir(self):
    return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)

  def __str__(self):
    return 'node_express'

# different configurations we can run under
with open('tools/run_tests/generated/configs.json') as f:
  _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))


_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'node_express': NodeExpressLanguage(),
    'php': PhpLanguage(),
    'php7': Php7Language(),
    'python': PythonLanguage(),
    'ruby': RubyLanguage(),
    'csharp': CSharpLanguage(),
    'objc' : ObjCLanguage(),
    'sanity': Sanity()
    }


_MSBUILD_CONFIG = {
    'dbg': 'Debug',
    'opt': 'Release',
    'gcov': 'Debug',
    }


def _windows_arch_option(arch):
  """Returns msbuild cmdline option for selected architecture."""
  if arch == 'default' or arch == 'x86':
    return '/p:Platform=Win32'
  elif arch == 'x64':
    return '/p:Platform=x64'
  else:
    print('Architecture %s not supported.' % arch)
    sys.exit(1)


def _check_arch_option(arch):
  """Checks that architecture option is valid."""
  if platform_string() == 'windows':
    _windows_arch_option(arch)
  elif platform_string() == 'linux':
    # On linux, we need to be running under docker with the right architecture.
    runtime_arch = platform.architecture()[0]
    if arch == 'default':
      return
    elif runtime_arch == '64bit' and arch == 'x64':
      return
    elif runtime_arch == '32bit' and arch == 'x86':
      return
    else:
      print('Architecture %s does not match current runtime architecture.' % arch)
      sys.exit(1)
  else:
    if args.arch != 'default':
      print('Architecture %s not supported on current platform.' % args.arch)
      sys.exit(1)


def _windows_build_bat(compiler):
  """Returns name of build.bat for selected compiler."""
  # For CoreCLR, fall back to the default compiler for C core
  if compiler == 'default' or compiler == 'vs2013':
    return 'vsprojects\\build_vs2013.bat'
  elif compiler == 'vs2015':
    return 'vsprojects\\build_vs2015.bat'
  else:
    print('Compiler %s not supported.' % compiler)
    sys.exit(1)


def _windows_toolset_option(compiler):
  """Returns msbuild PlatformToolset for selected compiler."""
  # For CoreCLR, fall back to the default compiler for C core
  if compiler == 'default' or compiler == 'vs2013' or compiler == 'coreclr':
    return '/p:PlatformToolset=v120'
  elif compiler == 'vs2015':
    return '/p:PlatformToolset=v140'
  else:
    print('Compiler %s not supported.' % compiler)
    sys.exit(1)


def _docker_arch_suffix(arch):
  """Returns suffix to dockerfile dir to use."""
  if arch == 'default' or arch == 'x64':
    return 'x64'
  elif arch == 'x86':
    return 'x86'
  else:
    print('Architecture %s not supported with current settings.' % arch)
    sys.exit(1)


def runs_per_test_type(arg_str):
    """Auxilary function to parse the "runs_per_test" flag.

       Returns:
           A positive integer or 0, the latter indicating an infinite number of
           runs.

       Raises:
           argparse.ArgumentTypeError: Upon invalid input.
    """
    if arg_str == 'inf':
        return 0
    try:
        n = int(arg_str)
        if n <= 0: raise ValueError
        return n
    except:
        msg = '\'{}\' is not a positive integer or \'inf\''.format(arg_str)
        raise argparse.ArgumentTypeError(msg)


def percent_type(arg_str):
  pct = float(arg_str)
  if pct > 100 or pct < 0:
    raise argparse.ArgumentTypeError(
        "'%f' is not a valid percentage in the [0, 100] range" % pct)
  return pct

# This is math.isclose in python >= 3.5
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
      return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)


# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=sorted(_CONFIGS.keys()),
                  default='opt')
argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
        help='A positive integer or "inf". If "inf", all tests will run in an '
             'infinite loop. Especially useful in combination with "-f"')
argp.add_argument('-r', '--regex', default='.*', type=str)
argp.add_argument('--regex_exclude', default='', type=str)
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
argp.add_argument('-s', '--slowdown', default=1.0, type=float)
argp.add_argument('-p', '--sample_percent', default=100.0, type=percent_type,
                  help='Run a random sample with that percentage of tests')
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-t', '--travis',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=['all'] + sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=['all'])
argp.add_argument('-S', '--stop_on_failure',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--use_docker',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Run all the tests under docker. That provides ' +
                  'additional isolation and prevents the need to install ' +
                  'language specific prerequisites. Only available on Linux.')
argp.add_argument('--allow_flakes',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
argp.add_argument('--arch',
                  choices=['default', 'x86', 'x64'],
                  default='default',
                  help='Selects architecture to target. For some platforms "default" is the only supported choice.')
argp.add_argument('--compiler',
                  choices=['default',
                           'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc_musl',
                           'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7',
                           'vs2013', 'vs2015',
                           'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine',
                           'node0.12', 'node4', 'node5', 'node6', 'node7', 'node8',
                           'electron1.3', 'electron1.6',
                           'coreclr',
                           'cmake'],
                  default='default',
                  help='Selects compiler to use. Allowed values depend on the platform and language.')
argp.add_argument('--iomgr_platform',
                  choices=['native', 'uv'],
                  default='native',
                  help='Selects iomgr platform to build on')
argp.add_argument('--build_only',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Perform all the build steps but don\'t run any tests.')
argp.add_argument('--measure_cpu_costs', default=False, action='store_const', const=True,
                  help='Measure the cpu costs of tests')
argp.add_argument('--update_submodules', default=[], nargs='*',
                  help='Update some submodules before building. If any are updated, also run generate_projects. ' +
                       'Submodules are specified as SUBMODULE_NAME:BRANCH; if BRANCH is omitted, master is assumed.')
argp.add_argument('-a', '--antagonists', default=0, type=int)
argp.add_argument('-x', '--xml_report', default=None, type=str,
        help='Generates a JUnit-compatible XML report')
argp.add_argument('--report_suite_name', default='tests', type=str,
        help='Test suite name to use in generated JUnit XML report')
argp.add_argument('--quiet_success',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' +
                       'Useful when running many iterations of each test (argument -n).')
argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
                  help='Don\'t try to iterate over many polling strategies when they exist')
argp.add_argument('--max_time', default=-1, type=int, help='Maximum test runtime in seconds')
argp.add_argument('--bq_result_table',
                  default='',
                  type=str,
                  nargs='?',
                  help='Upload test results to a specified BQ table.')
argp.add_argument('--auto_set_flakes', default=True, type=bool,
                  help='Set flakiness data from historic data')
args = argp.parse_args()

flaky_tests = set()
if args.auto_set_flakes:
  try:
    flaky_tests = set(get_flaky_tests())
  except:
    print("Unexpected error getting flaky tests:", sys.exc_info()[0])

if args.force_default_poller:
  _POLLING_STRATEGIES = {}

jobset.measure_cpu_costs = args.measure_cpu_costs

# update submodules if necessary
need_to_regenerate_projects = False
for spec in args.update_submodules:
  spec = spec.split(':', 1)
  if len(spec) == 1:
    submodule = spec[0]
    branch = 'master'
  elif len(spec) == 2:
    submodule = spec[0]
    branch = spec[1]
  cwd = 'third_party/%s' % submodule
  def git(cmd, cwd=cwd):
    print('in %s: git %s' % (cwd, cmd))
    run_shell_command('git %s' % cmd, cwd=cwd)
  git('fetch')
  git('checkout %s' % branch)
  git('pull origin %s' % branch)
  if os.path.exists('src/%s/gen_build_yaml.py' % submodule):
    need_to_regenerate_projects = True
if need_to_regenerate_projects:
  if jobset.platform_string() == 'linux':
    run_shell_command('tools/buildgen/generate_projects.sh')
  else:
    print('WARNING: may need to regenerate projects, but since we are not on')
    print('         Linux this step is being skipped. Compilation MAY fail.')


# grab config
run_config = _CONFIGS[args.config]
build_config = run_config.build_config

if args.travis:
  _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}

if 'all' in args.language:
  lang_list = _LANGUAGES.keys()
else:
  lang_list = args.language
# We don't support code coverage on some languages
if 'gcov' in args.config:
  for bad in ['objc', 'sanity']:
    if bad in lang_list:
      lang_list.remove(bad)

languages = set(_LANGUAGES[l] for l in lang_list)
for l in languages:
  l.configure(run_config, args)

language_make_options=[]
if any(language.make_options() for language in languages):
  if not 'gcov' in args.config and len(languages) != 1:
    print('languages with custom make options cannot be built simultaneously with other languages')
    sys.exit(1)
  else:
    # Combining make options is not clean and just happens to work. It allows C/C++ and C# to build
    # together, and is only used under gcov. All other configs should build languages individually.
    language_make_options = list(set([make_option for lang in languages for make_option in lang.make_options()]))

if args.use_docker:
  if not args.travis:
    print('Seen --use_docker flag, will run tests under docker.')
    print('')
    print('IMPORTANT: The changes you are testing need to be locally committed')
    print('because only the committed changes in the current branch will be')
    print('copied to the docker environment.')
    time.sleep(5)

  dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
  if len(dockerfile_dirs) > 1:
    if 'gcov' in args.config:
      dockerfile_dir = 'tools/dockerfile/test/multilang_jessie_x64'
      print ('Using multilang_jessie_x64 docker image for code coverage for '
             'all languages.')
    else:
      print ('Languages to be tested require running under different docker '
             'images.')
      sys.exit(1)
  else:
    dockerfile_dir = next(iter(dockerfile_dirs))

  child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ]
  run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])

  env = os.environ.copy()
  env['RUN_TESTS_COMMAND'] = run_tests_cmd
  env['DOCKERFILE_DIR'] = dockerfile_dir
  env['DOCKER_RUN_SCRIPT'] = 'tools/run_tests/dockerize/docker_run_tests.sh'
  if args.xml_report:
    env['XML_REPORT'] = args.xml_report
  if not args.travis:
    env['TTY_FLAG'] = '-t'  # enables Ctrl-C when not on Jenkins.

  subprocess.check_call('tools/run_tests/dockerize/build_docker_and_run_tests.sh',
                        shell=True,
                        env=env)
  sys.exit(0)

_check_arch_option(args.arch)

def make_jobspec(cfg, targets, makefile='Makefile'):
  if platform_string() == 'windows':
    if makefile.startswith('cmake/build/'):
      return [jobset.JobSpec(['cmake', '--build', '.',
                              '--target', '%s' % target,
                              '--config', _MSBUILD_CONFIG[cfg]],
                             cwd=os.path.dirname(makefile),
                             timeout_seconds=None) for target in targets]
    extra_args = []
    # better do parallel compilation
    # empirically /m:2 gives the best performance/price and should prevent
    # overloading the windows workers.
    extra_args.extend(['/m:2'])
    # disable PDB generation: it's broken, and we don't need it during CI
    extra_args.extend(['/p:Jenkins=true'])
    return [
      jobset.JobSpec([_windows_build_bat(args.compiler),
                      'vsprojects\\%s.sln' % target,
                      '/p:Configuration=%s' % _MSBUILD_CONFIG[cfg]] +
                      extra_args +
                      language_make_options,
                      shell=True, timeout_seconds=None)
      for target in targets]
  else:
    if targets and makefile.startswith('cmake/build/'):
      # With cmake, we've passed all the build configuration in the pre-build step already
      return [jobset.JobSpec([os.getenv('MAKE', 'make'),
                              '-j', '%d' % args.jobs] +
                             targets,
                             cwd='cmake/build',
                             timeout_seconds=None)]
    if targets:
      return [jobset.JobSpec([os.getenv('MAKE', 'make'),
                              '-f', makefile,
                              '-j', '%d' % args.jobs,
                              'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown,
                              'CONFIG=%s' % cfg,
                              'Q='] +
                              language_make_options +
                             ([] if not args.travis else ['JENKINS_BUILD=1']) +
                             targets,
                             timeout_seconds=None)]
    else:
      return []

make_targets = {}
for l in languages:
  makefile = l.makefile_name()
  make_targets[makefile] = make_targets.get(makefile, set()).union(
      set(l.make_targets()))

def build_step_environ(cfg):
  environ = {'CONFIG': cfg}
  msbuild_cfg = _MSBUILD_CONFIG.get(cfg)
  if msbuild_cfg:
    environ['MSBUILD_CONFIG'] = msbuild_cfg
  return environ

build_steps = list(set(
                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
                   for l in languages
                   for cmdline in l.pre_build_steps()))
if make_targets:
  make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.items())
  build_steps.extend(set(make_commands))
build_steps.extend(set(
                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
                   for l in languages
                   for cmdline in l.build_steps()))

post_tests_steps = list(set(
                        jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
                        for l in languages
                        for cmdline in l.post_tests_steps()))
runs_per_test = args.runs_per_test
forever = args.forever


def _shut_down_legacy_server(legacy_server_port):
  try:
    version = int(urllib.request.urlopen(
        'http://localhost:%d/version_number' % legacy_server_port,
        timeout=10).read())
  except:
    pass
  else:
    urllib.request.urlopen(
        'http://localhost:%d/quitquitquit' % legacy_server_port).read()


def _calculate_num_runs_failures(list_of_results):
  """Caculate number of runs and failures for a particular test.

  Args:
    list_of_results: (List) of JobResult object.
  Returns:
    A tuple of total number of runs and failures.
  """
  num_runs = len(list_of_results)  # By default, there is 1 run per JobResult.
  num_failures = 0
  for jobresult in list_of_results:
    if jobresult.retries > 0:
      num_runs += jobresult.retries
    if jobresult.num_failures > 0:
      num_failures += jobresult.num_failures
  return num_runs, num_failures


# _build_and_run results
class BuildAndRunError(object):

  BUILD = object()
  TEST = object()
  POST_TEST = object()


def _has_epollexclusive():
  try:
    subprocess.check_call('bins/%s/check_epollexclusive' % args.config)
    return True
  except subprocess.CalledProcessError, e:
    return False
  except OSError, e:
    # For languages other than C and Windows the binary won't exist
    return False


# returns a list of things that failed (or an empty list on success)
def _build_and_run(
    check_cancelled, newline_on_success, xml_report=None, build_only=False):
  """Do one pass of building & running tests."""
  # build latest sequentially
  num_failures, resultset = jobset.run(
      build_steps, maxjobs=1, stop_on_failure=True,
      newline_on_success=newline_on_success, travis=args.travis)
  if num_failures:
    return [BuildAndRunError.BUILD]

  if build_only:
    if xml_report:
      report_utils.render_junit_xml_report(resultset, xml_report,
                                           suite_name=args.report_suite_name)
    return []

  if not args.travis and not _has_epollexclusive() and platform_string() in _POLLING_STRATEGIES and 'epollex' in _POLLING_STRATEGIES[platform_string()]:
    print('\n\nOmitting EPOLLEXCLUSIVE tests\n\n')
    _POLLING_STRATEGIES[platform_string()].remove('epollex')

  # start antagonists
  antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py'])
                 for _ in range(0, args.antagonists)]
  start_port_server.start_port_server()
  resultset = None
  num_test_failures = 0
  try:
    infinite_runs = runs_per_test == 0
    one_run = set(
      spec
      for language in languages
      for spec in language.test_specs()
      if (re.search(args.regex, spec.shortname) and
          (args.regex_exclude == '' or
           not re.search(args.regex_exclude, spec.shortname))))
    # When running on travis, we want out test runs to be as similar as possible
    # for reproducibility purposes.
    if args.travis and args.max_time <= 0:
      massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
    else:
      # whereas otherwise, we want to shuffle things up to give all tests a
      # chance to run.
      massaged_one_run = list(one_run)  # random.sample needs an indexable seq.
      num_jobs = len(massaged_one_run)
      # for a random sample, get as many as indicated by the 'sample_percent'
      # argument. By default this arg is 100, resulting in a shuffle of all
      # jobs.
      sample_size = int(num_jobs * args.sample_percent/100.0)
      massaged_one_run = random.sample(massaged_one_run, sample_size)
      if not isclose(args.sample_percent, 100.0):
        assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)."
        print("Running %d tests out of %d (~%d%%)" %
              (sample_size, num_jobs, args.sample_percent))
    if infinite_runs:
      assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run'
    runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs
                     else itertools.repeat(massaged_one_run, runs_per_test))
    all_runs = itertools.chain.from_iterable(runs_sequence)

    if args.quiet_success:
      jobset.message('START', 'Running tests quietly, only failing tests will be reported', do_newline=True)
    num_test_failures, resultset = jobset.run(
        all_runs, check_cancelled, newline_on_success=newline_on_success,
        travis=args.travis, maxjobs=args.jobs,
        stop_on_failure=args.stop_on_failure,
        quiet_success=args.quiet_success, max_time=args.max_time)
    if resultset:
      for k, v in sorted(resultset.items()):
        num_runs, num_failures = _calculate_num_runs_failures(v)
        if num_failures > 0:
          if num_failures == num_runs:  # what about infinite_runs???
            jobset.message('FAILED', k, do_newline=True)
          else:
            jobset.message(
                'FLAKE', '%s [%d/%d runs flaked]' % (k, num_failures, num_runs),
                do_newline=True)
  finally:
    for antagonist in antagonists:
      antagonist.kill()
    if args.bq_result_table and resultset:
      upload_results_to_bq(resultset, args.bq_result_table, args, platform_string())
    if xml_report and resultset:
      report_utils.render_junit_xml_report(resultset, xml_report,
                                           suite_name=args.report_suite_name)

  number_failures, _ = jobset.run(
      post_tests_steps, maxjobs=1, stop_on_failure=True,
      newline_on_success=newline_on_success, travis=args.travis)

  out = []
  if number_failures:
    out.append(BuildAndRunError.POST_TEST)
  if num_test_failures:
    out.append(BuildAndRunError.TEST)

  return out


if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test', 'examples'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    errors = _build_and_run(check_cancelled=have_files_changed,
                            newline_on_success=False,
                            build_only=args.build_only) == 0
    if not previous_success and not errors:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    while not have_files_changed():
      time.sleep(1)
else:
  errors = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          xml_report=args.xml_report,
                          build_only=args.build_only)
  if not errors:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  exit_code = 0
  if BuildAndRunError.BUILD in errors:
    exit_code |= 1
  if BuildAndRunError.TEST in errors:
    exit_code |= 2
  if BuildAndRunError.POST_TEST in errors:
    exit_code |= 4
  sys.exit(exit_code)