Skip to content

Commit

Permalink
test: runner support for flaky tests
Browse files Browse the repository at this point in the history
Adding --flaky-tests option, to allow regarding flaky tests failures
as non-fatal.

Currently only observed by the TapProgressIndicator, which will
add a # TODO directive to tests classified as flaky. According to the
TAP specification, the test harness is supposed to treat failures
that have a # TODO directive as non-fatal.

Ported from nodejs/node-v0.x-archive@df3a2b2

PR-URL: #2424
Reviewed-By: Ben Noordhuis <[email protected]>
Reviewed-By: João Reis <[email protected]>
Reviewed-By: Sakthipriyan Vairamani <[email protected]>
  • Loading branch information
orangemocha authored and rvagg committed Aug 28, 2015
1 parent 1bc4468 commit 0cfd3be
Showing 1 changed file with 30 additions and 14 deletions.
44 changes: 30 additions & 14 deletions tools/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,9 @@

class ProgressIndicator(object):

def __init__(self, cases):
def __init__(self, cases, flaky_tests_mode):
self.cases = cases
self.flaky_tests_mode = flaky_tests_mode
self.parallel_queue = Queue(len(cases))
self.sequential_queue = Queue(len(cases))
for case in cases:
Expand Down Expand Up @@ -251,7 +252,10 @@ def HasRun(self, output):
self._done += 1
command = basename(output.command[-1])
if output.UnexpectedOutput():
logger.info('not ok %i - %s' % (self._done, command))
status_line = 'not ok %i - %s' % (self._done, command)
if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
status_line = status_line + ' # TODO : Fix flaky test'
logger.info(status_line)
for l in output.output.stderr.splitlines():
logger.info('#' + l)
for l in output.output.stdout.splitlines():
Expand All @@ -262,7 +266,10 @@ def HasRun(self, output):
logger.info(
'ok %i - %s # skip %s' % (self._done, command, skip.group(1)))
else:
logger.info('ok %i - %s' % (self._done, command))
status_line = 'ok %i - %s' % (self._done, command)
if FLAKY in output.test.outcomes:
status_line = status_line + ' # TODO : Fix flaky test'
logger.info(status_line)

duration = output.test.duration

Expand All @@ -280,8 +287,8 @@ def Done(self):

class CompactProgressIndicator(ProgressIndicator):

def __init__(self, cases, templates):
super(CompactProgressIndicator, self).__init__(cases)
def __init__(self, cases, flaky_tests_mode, templates):
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
self.templates = templates
self.last_status_length = 0
self.start_time = time.time()
Expand Down Expand Up @@ -336,29 +343,29 @@ def PrintProgress(self, name):

class ColorProgressIndicator(CompactProgressIndicator):

def __init__(self, cases):
def __init__(self, cases, flaky_tests_mode):
templates = {
'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
'stdout': "\033[1m%s\033[0m",
'stderr': "\033[31m%s\033[0m",
}
super(ColorProgressIndicator, self).__init__(cases, templates)
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)

def ClearLine(self, last_line_length):
print "\033[1K\r",


class MonochromeProgressIndicator(CompactProgressIndicator):

def __init__(self, cases):
def __init__(self, cases, flaky_tests_mode):
templates = {
'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
'stdout': '%s',
'stderr': '%s',
'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
'max_length': 78
}
super(MonochromeProgressIndicator, self).__init__(cases, templates)
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)

def ClearLine(self, last_line_length):
print ("\r" + (" " * last_line_length) + "\r"),
Expand Down Expand Up @@ -780,8 +787,8 @@ def GetVmFlags(self, testcase, mode):
def GetTimeout(self, mode):
return self.timeout * TIMEOUT_SCALEFACTOR[ARCH_GUESS or 'ia32'][mode]

def RunTestCases(cases_to_run, progress, tasks):
progress = PROGRESS_INDICATORS[progress](cases_to_run)
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
return progress.Run(tasks)


Expand All @@ -805,7 +812,8 @@ def BuildRequirements(context, requirements, mode, scons_flags):
TIMEOUT = 'timeout'
CRASH = 'crash'
SLOW = 'slow'

FLAKY = 'flaky'
DONTCARE = 'dontcare'

class Expression(object):
pass
Expand Down Expand Up @@ -1253,6 +1261,9 @@ def BuildOptions():
default=False, action="store_true")
result.add_option("--cat", help="Print the source of the tests",
default=False, action="store_true")
result.add_option("--flaky-tests",
help="Regard tests marked as flaky (run|skip|dontcare)",
default="run")
result.add_option("--warn-unused", help="Report unused rules",
default=False, action="store_true")
result.add_option("-j", help="The number of parallel tasks to run",
Expand Down Expand Up @@ -1303,6 +1314,9 @@ def ProcessOptions(options):
return False
if options.J:
options.j = multiprocessing.cpu_count()
if options.flaky_tests not in ["run", "skip", "dontcare"]:
print "Unknown flaky-tests mode %s" % options.flaky_tests
return False
return True


Expand Down Expand Up @@ -1505,7 +1519,9 @@ def Main():

result = None
def DoSkip(case):
return SKIP in case.outcomes or SLOW in case.outcomes
if SKIP in case.outcomes or SLOW in case.outcomes:
return True
return FLAKY in case.outcomes and options.flaky_tests == SKIP
cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
if options.run is not None:
# Must ensure the list of tests is sorted before selecting, to avoid
Expand All @@ -1522,7 +1538,7 @@ def DoSkip(case):
else:
try:
start = time.time()
if RunTestCases(cases_to_run, options.progress, options.j):
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
result = 0
else:
result = 1
Expand Down

0 comments on commit 0cfd3be

Please sign in to comment.