Commit 5eb1d6d3 authored by Mathieu Giraud's avatar Mathieu Giraud

Merge branch 'feature-14-and-refactor' into 'dev'

Feature 14 and refactor

Closes #14

See merge request !9
parents 2eff7932 e5dcfcb5
Pipeline #618 passed with stages
in 12 seconds
......@@ -7,7 +7,7 @@ $ The command is launched with 'echo'
: LC_ALL=C cal 2001
$ The command is not run, several tests do fail
0: January
0: October
: failed
......@@ -109,8 +109,19 @@ STATUS = {
ALLOW_FAILED: 'failed-but-ALLOW',
}
STATUS_ORDER = [
# Failed
False, TODO_PASSED,
# Warnings
TODO, ALLOW_FAILED, SKIP,
# Passed
True,
# 'Forgotten' status when mixed to other tests
None
]
FAIL_STATUS = [False, TODO_PASSED]
WARN_STATUS = FAIL_STATUS + [SKIP, TODO]
WARN_STATUS = FAIL_STATUS + [ALLOW_FAILED, TODO, SKIP]
STATUS_TAP = {
None: 'not run',
......@@ -127,6 +138,22 @@ STATUS_XML[False] = 'failure'
STATUS_XML[SKIP] = 'skipped'
def combine_status(s1, s2):
'''
>>> combine_status(TODO, False)
False
>>> combine_status(True, SKIP) == SKIP
True
>>> combine_status(True, TODO_PASSED) == TODO_PASSED
True
'''
i1 = STATUS_ORDER.index(s1)
i2 = STATUS_ORDER.index(s2)
return STATUS_ORDER[min(i1,i2)]
# Simple colored output
CSIm = '\033[%sm'
......@@ -618,12 +645,7 @@ class TestCase(TestCaseAbstract):
class TestSuite():
'''
>>> s = TestSuite()
>>> s.load(['echo "hello"', '$My test', ':hello'])
>>> print(s)
echo "hello"
My test
>>> s.test()
>>> s.test(['echo "hello"', '$My test', ':hello'], colorize = False)
True
>>> s.tests[0].status
True
......@@ -631,8 +653,7 @@ class TestSuite():
>>> s2 = TestSuite('r')
>>> s2.variables.append(("$LAUNCHER", ""))
>>> s2.variables.append(("$EXTRA", ""))
>>> s2.load(['echo "hello"', '$ A nice test', ':e.*o'])
>>> s2.test(verbose = 1, colorize = False) # doctest: +NORMALIZE_WHITESPACE
>>> s2.test(['echo "hello"', '$ A nice test', ':e.*o'], verbose = 1, colorize = False) # doctest: +NORMALIZE_WHITESPACE
echo "hello"
stdout --> 1 lines
stderr --> 0 lines
......@@ -670,7 +691,15 @@ class TestSuite():
self.elapsed_time = None
self.timeout = timeout
def load(self, should_lines):
def cmd_variables_cd(self, cmd, verbose, colorize):
cmd = replace_variables(cmd, self.variables_all)
if self.cd:
cmd = 'cd %s ; ' % self.cd + cmd
if verbose > 0:
print(color(ANSI.MAGENTA, cmd, colorize))
return cmd
def test(self, should_lines, variables=[], verbose=0, colorize=True, only=None):
name = ''
this_cmd_continues = False
for l in should_lines:
......@@ -750,38 +779,16 @@ class TestSuite():
this_cmd_continues = next_cmd_continues
def print_stderr(self, colorize=True):
print(' stdout --> %s lines' % len(self.stdout))
print(' stderr --> %s lines' % len(self.stderr))
print(color(ANSI.CYAN, ''.join(self.stderr), colorize))
def skip_all(self, reason, verbose=1):
if verbose > 0:
print('Skipping tests: %s' % reason)
for test in self.tests:
test.status = SKIP
self.stats.up(test.status)
self.status = SKIP
def test(self, variables=[], verbose=0, colorize=True, only=None):
# Test
self.only = only
self.variables_all = self.variables + variables
if verbose > 1:
print_variables(self.variables_all)
def cmd_variables_cd(cmd):
cmd = replace_variables(cmd, self.variables_all)
if self.cd:
cmd = 'cd %s ; ' % self.cd + cmd
if verbose > 0:
print(color(ANSI.MAGENTA, cmd, colorize))
return cmd
self.status = True
self.status = None
if self.requires_cmd:
requires_cmd = cmd_variables_cd(self.requires_cmd)
requires_cmd = self.cmd_variables_cd(self.requires_cmd, verbose, colorize)
p = subprocess.Popen(requires_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
self.requires = (p.wait() == 0)
self.requires_stderr = [l.decode(errors='replace') for l in p.stderr.readlines()]
......@@ -797,10 +804,18 @@ class TestSuite():
self.skip_all('%s while %s is given' % (DIRECTIVE_NO_LAUNCHER, VAR_LAUNCHER), verbose)
return self.status
self.test_lines += self.launch(self.cmds, verbose, colorize)
self.tests_on_lines(self.tests, self.test_lines, verbose, colorize)
self.debug(self.status, "\n".join(self.cmds), self.test_lines, verbose, colorize)
return self.status
def launch(self, cmds, verbose, colorize):
start_time = time.time()
cmd = ' ; '.join(map(pre_process, self.cmds))
cmd = cmd_variables_cd(cmd)
cmd = ' ; '.join(map(pre_process, cmds))
cmd = self.cmd_variables_cd(cmd, verbose, colorize)
f_stdout = tempfile.TemporaryFile()
f_stderr = tempfile.TemporaryFile()
......@@ -816,6 +831,10 @@ class TestSuite():
self.tests.append(ExternalTestCase('Exit code is %d' % self.expected_exit_code, SKIP, 'timeout after %s seconds' % self.timeout))
p.kill()
if self.elapsed_time is None:
self.elapsed_time = 0
self.elapsed_time += time.time() - start_time
f_stdout.seek(0)
f_stderr.seek(0)
self.stdout = [l.decode(errors='replace') for l in f_stdout.readlines()]
......@@ -823,52 +842,64 @@ class TestSuite():
f_stdout.close()
f_stderr.close()
if verbose > 0:
self.print_stderr(colorize)
self.test_lines = open(self.source).readlines() if self.source else self.stdout
return open(self.source).readlines() if self.source else self.stdout
for test in self.tests:
def tests_on_lines(self, tests, test_lines, verbose, colorize):
'''
Test all tests in 'tests' on 'test_lines',
taking into accound self modifiers
and gathering statuses in self.status
'''
for test in tests:
# Filter
if only:
if not only(test):
if self.only:
if not self.only(test):
test.status = SKIP
continue
# Test the test
test.test(self.test_lines, variables=self.variables_all, verbose=verbose-1)
test.test(test_lines, variables=self.variables_all, verbose=verbose-1)
self.stats.up(test.status)
# When a test fails, the file fails
if test.status in FAIL_STATUS:
self.status = False
# When the file is not failing, we may report a more sublte status
if test.status in WARN_STATUS and self.status is True:
self.status = test.status
self.status = combine_status(self.status, test.status)
if verbose > 0 or test.status in WARN_STATUS:
print(test.str(colorize))
if self.status is False and verbose <= 0:
def print_stderr(self, colorize=True):
print(' stdout --> %s lines' % len(self.stdout))
print(' stderr --> %s lines' % len(self.stderr))
print(color(ANSI.CYAN, ''.join(self.stderr), colorize))
def skip_all(self, reason, verbose=1):
if verbose > 0:
print('Skipping tests: %s' % reason)
for test in self.tests:
test.status = SKIP
self.stats.up(test.status)
self.status = SKIP
def debug(self, status, cmd, test_lines, verbose, colorize):
if status in FAIL_STATUS and verbose <= 0:
print(color(ANSI.MAGENTA, cmd, colorize))
self.print_stderr(colorize)
if self.status is False or verbose > 1:
if status in FAIL_STATUS or verbose > 1:
print(LINE)
if len(self.test_lines) <= MAX_DUMP_LINES:
print(''.join(self.test_lines), end='')
if len(test_lines) <= MAX_DUMP_LINES:
print(''.join(test_lines), end='')
else:
print(''.join(self.test_lines[:MAX_HALF_DUMP_LINES]), end='')
print(color(ANSI.MAGENTA, '... %d other lines ...' % (len(self.test_lines) - 2*MAX_HALF_DUMP_LINES), colorize))
print(''.join(self.test_lines[-MAX_HALF_DUMP_LINES:]), end='')
print(''.join(test_lines[:MAX_HALF_DUMP_LINES]), end='')
print(color(ANSI.MAGENTA, '... %d other lines ...' % (len(test_lines) - 2*MAX_HALF_DUMP_LINES), colorize))
print(''.join(test_lines[-MAX_HALF_DUMP_LINES:]), end='')
print(LINE)
self.elapsed_time = time.time() - start_time
return self.status
def str_status(self, colorize=True):
return self.stats.str_status(self.status, colorize)
......@@ -931,7 +962,7 @@ class FileSet():
return len(self.files)
def test(self, variables=None, cd=None, cd_same=False, output=None, verbose=0, only=None):
self.status = True
self.status = None
try:
for f in self.files:
......@@ -940,13 +971,9 @@ class FileSet():
cd_f = os.path.dirname(f) if cd_same else cd
s = TestSuite(self.modifiers, cd_f, name = f, timeout = self.timeout)
self.sets.append(s)
s.load(open(f))
s.test(variables, verbose - 1, only=only)
s.test(open(f), variables, verbose - 1, only=only)
self.stats.up(s.status, f)
if not s.status:
self.status = False
self.status = combine_status(self.status, s.status)
self.stats_tests += s.stats
filename_without_ext = os.path.splitext(f)[0]
......@@ -1068,6 +1095,6 @@ if __name__ == '__main__':
if len(fs) > 1:
retry = fs.write_retry(sys.argv[1:], args.file, verbose = args.verbose)
sys.exit(0 if status else 1)
sys.exit(1 if status in FAIL_STATUS else 0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment