run_tests.py

#!/usr/bin/python2.7
"""Run tests in parallel."""

import argparse
import glob
import itertools
import json
import multiprocessing
import os
import sys
import time

import jobset
import watch_dirs


# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class SimpleConfig(object):

  def __init__(self, config, environ={}):
    self.build_config = config
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = (config != 'gcov')
    self.environ = environ

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=[binary],
                          environ=self.environ,
                          hash_targets=hash_targets
                              if self.allow_hashing else None)


# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
class ValgrindConfig(object):

  def __init__(self, config, tool):
    self.build_config = config
    self.tool = tool
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = False

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool, binary],
                   hash_targets=None)


class CLanguage(object):

  def __init__(self, make_target, test_lang):
    self.make_target = make_target
    with open('tools/run_tests/tests.json') as f:
      js = json.load(f)
      self.binaries = [tgt['name']
                       for tgt in js
                       if tgt['language'] == test_lang]

  def test_specs(self, config):
    out = []
    for name in self.binaries:
      binary = 'bins/%s/%s' % (config.build_config, name)
      out.append(config.job_spec(binary, [binary]))
    return out

  def make_targets(self):
    return ['buildtests_%s' % self.make_target]

  def build_steps(self):
    return []


class NodeLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_node.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_node.sh']]


class PhpLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('src/php/bin/run_tests.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_php.sh']]


class PythonLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_python.sh', None)]

  def make_targets(self):
    return[]

  def build_steps(self):
    return [['tools/run_tests/build_python.sh']]


# different configurations we can run under
_CONFIGS = {
    'dbg': SimpleConfig('dbg'),
    'opt': SimpleConfig('opt'),
    'tsan': SimpleConfig('tsan', environ={
        'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt'}),
    'msan': SimpleConfig('msan'),
    'ubsan': SimpleConfig('ubsan'),
    'asan': SimpleConfig('asan', environ={
        'ASAN_OPTIONS': 'detect_leaks=1:color=always:suppressions=tools/tsan_suppressions.txt'}),
    'gcov': SimpleConfig('gcov'),
    'memcheck': ValgrindConfig('valgrind', 'memcheck'),
    'helgrind': ValgrindConfig('dbg', 'helgrind')
    }


_DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'php': PhpLanguage(),
    'python': PythonLanguage(),
    }

# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=['all'] + sorted(_CONFIGS.keys()),
                  nargs='+',
                  default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=int)
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=sorted(_LANGUAGES.keys()))
args = argp.parse_args()

# grab config
run_configs = set(_CONFIGS[cfg]
                  for cfg in itertools.chain.from_iterable(
                      _CONFIGS.iterkeys() if x == 'all' else [x]
                      for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)

make_targets = []
languages = set(_LANGUAGES[l] for l in args.language)
build_steps = [jobset.JobSpec(['make',
                               '-j', '%d' % (multiprocessing.cpu_count() + 1),
                               'CONFIG=%s' % cfg] + list(set(
                                   itertools.chain.from_iterable(
                                       l.make_targets() for l in languages))))
               for cfg in build_configs] + list(set(
                   jobset.JobSpec(cmdline)
                   for l in languages
                   for cmdline in l.build_steps()))
one_run = set(
    spec
    for config in run_configs
    for language in args.language
    for spec in _LANGUAGES[language].test_specs(config))

runs_per_test = args.runs_per_test
forever = args.forever


class TestCache(object):
  """Cache for running tests."""

  def __init__(self, use_cache_results):
    self._last_successful_run = {}
    self._use_cache_results = use_cache_results

  def should_run(self, cmdline, bin_hash):
    if cmdline not in self._last_successful_run:
      return True
    if self._last_successful_run[cmdline] != bin_hash:
      return True
    if not self._use_cache_results:
      return True
    return False

  def finished(self, cmdline, bin_hash):
    self._last_successful_run[cmdline] = bin_hash

  def dump(self):
    return [{'cmdline': k, 'hash': v}
            for k, v in self._last_successful_run.iteritems()]

  def parse(self, exdump):
    self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump)

  def save(self):
    with open('.run_tests_cache', 'w') as f:
      f.write(json.dumps(self.dump()))

  def maybe_load(self):
    if os.path.exists('.run_tests_cache'):
      with open('.run_tests_cache') as f:
        self.parse(json.loads(f.read()))


def _build_and_run(check_cancelled, newline_on_success, cache):
  """Do one pass of building & running tests."""
  # build latest sequentially
  if not jobset.run(build_steps, maxjobs=1):
    return 1

  # run all the tests
  all_runs = itertools.chain.from_iterable(
      itertools.repeat(one_run, runs_per_test))
  if not jobset.run(all_runs, check_cancelled,
                    newline_on_success=newline_on_success,
                    maxjobs=min(c.maxjobs for c in run_configs),
                    cache=cache):
    return 2

  return 0


test_cache = TestCache(runs_per_test == 1)
test_cache.maybe_load()

if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    success = _build_and_run(check_cancelled=have_files_changed,
                             newline_on_success=False,
                             cache=test_cache) == 0
    if not previous_success and success:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    test_cache.save()
    while not have_files_changed():
      time.sleep(1)
else:
  result = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          cache=test_cache)
  if result == 0:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  test_cache.save()
  sys.exit(result)