diff --git a/AUTHORS b/AUTHORS index b4ec9078..64d7ff84 100644 --- a/AUTHORS +++ b/AUTHORS @@ -25,6 +25,7 @@ The following submitted code, packages or analysis, and deserve special thanks: A M asmyers Lukas Barth + Paul J. Fenwick Thanks to the following, who submitted detailed bug reports and excellent suggestions: diff --git a/ChangeLog b/ChangeLog index a5387ad0..04c72934 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ (Thanks to Jörg Krause, Ben Boeckel). - TW-1845 Cygwin build fails, missing get_current_dir_name (thanks to hosaka). +- TW-1936 Tweak tests to have fuller TAP compliance + (thanks to Paul J. Fenwick) - TI-27 Continue tracking by ID (thanks to Dennis Schubert) - TI-29 timew config can't add new value diff --git a/test/problems b/test/problems index 5bacda41..3e739422 100755 --- a/test/problems +++ b/test/problems @@ -54,6 +54,15 @@ if __name__ == "__main__": file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$") timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$") + + expected_fail = re.compile(r"^not ok.*?#\s*TODO", re.I) + unexpected_pass = re.compile(r"^ok .*?#\s*TODO", re.I) + skip = re.compile(r"^ok .*?#\s*skip", re.I) + ok = re.compile(r"^ok ", re.I) + not_ok = re.compile(r"^not ok", re.I) + comment = re.compile(r"^#") + plan = re.compile(r"^1..\d+\s*(?:#.*)?$") + start = None stop = None @@ -68,31 +77,37 @@ if __name__ == "__main__": if match: filename = match.group(1) - if line.startswith("ok "): - passed[filename] += 1 - - if line.startswith("not "): - errors[filename] += 1 - - if line.startswith("skip "): - skipped[filename] += 1 - - if line.startswith("# EXPECTED_FAILURE:"): + elif expected_fail.match(line): expected[filename] += 1 - if line.startswith("# UNEXPECTED_SUCCESS:"): + elif unexpected_pass.match(line): unexpected[filename] += 1 + elif skip.match(line): + skipped[filename] += 1 + + # It's important these come last, since they're subpatterns of the above + + elif ok.match(line): + passed[filename] += 1 + + elif not_ok.match(line): + errors[filename] += 1 + + elif comment.match(line): + pass + + elif plan.match(line): + pass + + else: + # Uncomment if you want to see malformed things we caught as well... + # print(color("Malformed TAP (" + filename + "): " + line, "red")) + pass + # Last line contains the ending timestamp stop = float(timestamp.match(line).group(1)) - # Remove expected failures from the skipped tests category - for filename, value in expected.items(): - if skipped[filename] == value: - del skipped[filename] - else: - skipped[filename] -= value - v = "{0:>5d}" passed_str = "Passed:" + pad(24) passed_int = v.format(sum(passed.values())) diff --git a/test/simpletap/__init__.py b/test/simpletap/__init__.py index 518d48c5..6ab4c7fa 100644 --- a/test/simpletap/__init__.py +++ b/test/simpletap/__init__.py @@ -1,4 +1,5 @@ ############################################################################### +# taskwarrior - a command line task list manager. # # Copyright 2006 - 2017, Paul Beckingham, Federico Hernandez. # @@ -154,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult): if status: if status == "SKIP": - self.stream.writeln("{0} {1} - {2}: {3}".format( - color("skip", "yellow"), self.testsRun, filename, desc) + self.stream.writeln("{0} {1} - {2}: {3} # skip".format( + color("ok", "yellow"), self.testsRun, filename, desc) ) elif status == "EXPECTED_FAILURE": - self.stream.writeln("{0} {1} - {2}: {3}".format( - color("skip", "yellow"), self.testsRun, filename, desc) + self.stream.writeln("{0} {1} - {2}: {3} # TODO".format( + color("not ok", "yellow"), self.testsRun, filename, desc) ) else: self.stream.writeln("{0} {1} - {2}: {3}".format( @@ -226,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner): unittest.signals.registerResult(result) result.failfast = self.failfast + # TAP requires output is on STDOUT. + # TODO: Define this at __init__ time + result.stream = unittest.runner._WritelnDecorator(sys.stdout) + with warnings.catch_warnings(): if getattr(self, "warnings", None): # if self.warnings is set, use it to filter all the warnings diff --git a/test/test.cpp b/test/test.cpp index 14dc1954..891f70c8 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -452,11 +452,12 @@ void UnitTest::skip (const std::string& text) { ++_counter; ++_skipped; - std::cout << yellow ("skip") + std::cout << yellow ("ok") << " " << _counter << " - " << text + << " # skip" << '\n'; }