Skip to content
Snippets Groups Projects
Commit 10ce383e authored by Jan Tattermusch's avatar Jan Tattermusch
Browse files

Merge pull request #3487 from ctiller/flaky-timeouts

Add support for flaky timeouts
parents 96bcf021 bfc8a06a
No related branches found
No related tags found
No related merge requests found
...@@ -83,6 +83,7 @@ _CLEAR_LINE = '\x1b[2K' ...@@ -83,6 +83,7 @@ _CLEAR_LINE = '\x1b[2K'
_TAG_COLOR = { _TAG_COLOR = {
'FAILED': 'red', 'FAILED': 'red',
'FLAKE': 'purple', 'FLAKE': 'purple',
'TIMEOUT_FLAKE': 'purple',
'WARNING': 'yellow', 'WARNING': 'yellow',
'TIMEOUT': 'red', 'TIMEOUT': 'red',
'PASSED': 'green', 'PASSED': 'green',
...@@ -133,7 +134,8 @@ class JobSpec(object): ...@@ -133,7 +134,8 @@ class JobSpec(object):
"""Specifies what to run for a job.""" """Specifies what to run for a job."""
def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None, def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0): cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0,
timeout_retries=0):
""" """
Arguments: Arguments:
cmdline: a list of arguments to pass as the command line cmdline: a list of arguments to pass as the command line
...@@ -153,6 +155,7 @@ class JobSpec(object): ...@@ -153,6 +155,7 @@ class JobSpec(object):
self.shell = shell self.shell = shell
self.timeout_seconds = timeout_seconds self.timeout_seconds = timeout_seconds
self.flake_retries = flake_retries self.flake_retries = flake_retries
self.timeout_retries = timeout_retries
def identity(self): def identity(self):
return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets) return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
...@@ -176,6 +179,7 @@ class Job(object): ...@@ -176,6 +179,7 @@ class Job(object):
self._xml_test = ET.SubElement(xml_report, 'testcase', self._xml_test = ET.SubElement(xml_report, 'testcase',
name=self._spec.shortname) if xml_report is not None else None name=self._spec.shortname) if xml_report is not None else None
self._retries = 0 self._retries = 0
self._timeout_retries = 0
message('START', spec.shortname, do_newline=self._travis) message('START', spec.shortname, do_newline=self._travis)
self.start() self.start()
...@@ -223,19 +227,26 @@ class Job(object): ...@@ -223,19 +227,26 @@ class Job(object):
ET.SubElement(self._xml_test, 'failure', message='Failure').text ET.SubElement(self._xml_test, 'failure', message='Failure').text
else: else:
self._state = _SUCCESS self._state = _SUCCESS
message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries), message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
do_newline=self._newline_on_success or self._travis) self._spec.shortname, elapsed, self._retries, self._timeout_retries),
do_newline=self._newline_on_success or self._travis)
if self._bin_hash: if self._bin_hash:
update_cache.finished(self._spec.identity(), self._bin_hash) update_cache.finished(self._spec.identity(), self._bin_hash)
elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds: elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
self._tempfile.seek(0) self._tempfile.seek(0)
stdout = self._tempfile.read() stdout = self._tempfile.read()
filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore')) filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
message('TIMEOUT', self._spec.shortname, stdout, do_newline=True) if self._timeout_retries < self._spec.timeout_retries:
self.kill() message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True)
if self._xml_test is not None: self._timeout_retries += 1
ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout self._process.terminate()
ET.SubElement(self._xml_test, 'error', message='Timeout') self.start()
else:
message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
self.kill()
if self._xml_test is not None:
ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
ET.SubElement(self._xml_test, 'error', message='Timeout')
return self._state return self._state
def kill(self): def kill(self):
......
...@@ -120,7 +120,8 @@ class ValgrindConfig(object): ...@@ -120,7 +120,8 @@ class ValgrindConfig(object):
self.args + cmdline, self.args + cmdline,
shortname='valgrind %s' % cmdline[0], shortname='valgrind %s' % cmdline[0],
hash_targets=None, hash_targets=None,
flake_retries=5 if args.allow_flakes else 0) flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0)
def get_c_tests(travis, test_lang) : def get_c_tests(travis, test_lang) :
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment