From 4baf7be64a9fed586f407df8d9fd0387f21e273a Mon Sep 17 00:00:00 2001 From: Paul Beckingham Date: Sat, 9 Dec 2017 23:19:14 -0500 Subject: [PATCH] TW-1936: Tweak tests to have fuller TAP compliance - Thanks to Paul J. Fenwick. --- ChangeLog | 2 ++ test/problems | 51 +++++++++++++++++++++++------------- test/simpletap/__init__.py | 13 +++++++--- test/test.cpp | 53 +++++++++++++++++++------------------- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/ChangeLog b/ChangeLog index ccd161f..a698044 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,8 @@ (thanks to Eric Hymowitz, Paul Fenwick). - TS-34 Tasksh throw a warning at the end of a review command (thanks to bjonnh). +- TW-1936 Tweak tests to have fuller TAP compliance + (thanks to Paul J. Fenwick) - Review report now defaults to 6 days instead of 1 weeķ, which is more convenient for those who review weekly (thanks to Dirk Deimeke). diff --git a/test/problems b/test/problems index 5bacda4..3e73942 100755 --- a/test/problems +++ b/test/problems @@ -54,6 +54,15 @@ if __name__ == "__main__": file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$") timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$") + + expected_fail = re.compile(r"^not ok.*?#\s*TODO", re.I) + unexpected_pass = re.compile(r"^ok .*?#\s*TODO", re.I) + skip = re.compile(r"^ok .*?#\s*skip", re.I) + ok = re.compile(r"^ok ", re.I) + not_ok = re.compile(r"^not ok", re.I) + comment = re.compile(r"^#") + plan = re.compile(r"^1..\d+\s*(?:#.*)?$") + start = None stop = None @@ -68,31 +77,37 @@ if __name__ == "__main__": if match: filename = match.group(1) - if line.startswith("ok "): - passed[filename] += 1 - - if line.startswith("not "): - errors[filename] += 1 - - if line.startswith("skip "): - skipped[filename] += 1 - - if line.startswith("# EXPECTED_FAILURE:"): + elif expected_fail.match(line): expected[filename] += 1 - if line.startswith("# UNEXPECTED_SUCCESS:"): + elif unexpected_pass.match(line): unexpected[filename] += 1 + elif skip.match(line): + skipped[filename] += 1 + + # It's important these come last, since they're subpatterns of the above + + elif ok.match(line): + passed[filename] += 1 + + elif not_ok.match(line): + errors[filename] += 1 + + elif comment.match(line): + pass + + elif plan.match(line): + pass + + else: + # Uncomment if you want to see malformed things we caught as well... + # print(color("Malformed TAP (" + filename + "): " + line, "red")) + pass + # Last line contains the ending timestamp stop = float(timestamp.match(line).group(1)) - # Remove expected failures from the skipped tests category - for filename, value in expected.items(): - if skipped[filename] == value: - del skipped[filename] - else: - skipped[filename] -= value - v = "{0:>5d}" passed_str = "Passed:" + pad(24) passed_int = v.format(sum(passed.values())) diff --git a/test/simpletap/__init__.py b/test/simpletap/__init__.py index 518d48c..6ab4c7f 100644 --- a/test/simpletap/__init__.py +++ b/test/simpletap/__init__.py @@ -1,4 +1,5 @@ ############################################################################### +# taskwarrior - a command line task list manager. # # Copyright 2006 - 2017, Paul Beckingham, Federico Hernandez. # @@ -154,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult): if status: if status == "SKIP": - self.stream.writeln("{0} {1} - {2}: {3}".format( - color("skip", "yellow"), self.testsRun, filename, desc) + self.stream.writeln("{0} {1} - {2}: {3} # skip".format( + color("ok", "yellow"), self.testsRun, filename, desc) ) elif status == "EXPECTED_FAILURE": - self.stream.writeln("{0} {1} - {2}: {3}".format( - color("skip", "yellow"), self.testsRun, filename, desc) + self.stream.writeln("{0} {1} - {2}: {3} # TODO".format( + color("not ok", "yellow"), self.testsRun, filename, desc) ) else: self.stream.writeln("{0} {1} - {2}: {3}".format( @@ -226,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner): unittest.signals.registerResult(result) result.failfast = self.failfast + # TAP requires output is on STDOUT. + # TODO: Define this at __init__ time + result.stream = unittest.runner._WritelnDecorator(sys.stdout) + with warnings.catch_warnings(): if getattr(self, "warnings", None): # if self.warnings is set, use it to filter all the warnings diff --git a/test/test.cpp b/test/test.cpp index 9dc7552..891f70c 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -51,7 +51,7 @@ UnitTest::UnitTest (int planned) , _failed (0) , _skipped (0) { - std::cout << "1.." << _planned << "\n"; + std::cout << "1.." << _planned << '\n'; } /////////////////////////////////////////////////////////////////////////////// @@ -99,14 +99,14 @@ void UnitTest::plan (int planned) _failed = 0; _skipped = 0; - std::cout << "1.." << _planned << "\n"; + std::cout << "1.." << _planned << '\n'; } /////////////////////////////////////////////////////////////////////////////// void UnitTest::planMore (int extra) { _planned += extra; - std::cout << "1.." << _planned << "\n"; + std::cout << "1.." << _planned << '\n'; } /////////////////////////////////////////////////////////////////////////////// @@ -122,7 +122,7 @@ void UnitTest::ok (bool expression, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -132,7 +132,7 @@ void UnitTest::ok (bool expression, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } } @@ -149,7 +149,7 @@ void UnitTest::notok (bool expression, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -159,7 +159,7 @@ void UnitTest::notok (bool expression, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } } @@ -175,7 +175,7 @@ void UnitTest::is (bool actual, bool expected, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -189,7 +189,7 @@ void UnitTest::is (bool actual, bool expected, const std::string& name) << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -205,7 +205,7 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -219,7 +219,7 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name) << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -235,7 +235,7 @@ void UnitTest::is (int actual, int expected, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -249,7 +249,7 @@ void UnitTest::is (int actual, int expected, const std::string& name) << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -265,7 +265,7 @@ void UnitTest::is (double actual, double expected, const std::string& name) << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -279,7 +279,7 @@ void UnitTest::is (double actual, double expected, const std::string& name) << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -295,7 +295,7 @@ void UnitTest::is (double actual, double expected, double tolerance, const std:: << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -309,7 +309,7 @@ void UnitTest::is (double actual, double expected, double tolerance, const std:: << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -325,7 +325,7 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -339,7 +339,7 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri << expected << "\n# got: " << actual - << "\n"; + << '\n'; } } @@ -358,7 +358,7 @@ void UnitTest::is ( << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -392,7 +392,7 @@ void UnitTest::is ( << _counter << " - " << name - << "\n"; + << '\n'; } else { @@ -418,7 +418,7 @@ void UnitTest::diag (const std::string& text) auto end = text.find_last_not_of (" \t\n\r\f"); if (start != std::string::npos && end != std::string::npos) - std::cout << "# " << text.substr (start, end - start + 1) << "\n"; + std::cout << "# " << text.substr (start, end - start + 1) << '\n'; } /////////////////////////////////////////////////////////////////////////////// @@ -431,7 +431,7 @@ void UnitTest::pass (const std::string& text) << _counter << " - " << text - << "\n"; + << '\n'; } /////////////////////////////////////////////////////////////////////////////// @@ -444,7 +444,7 @@ void UnitTest::fail (const std::string& text) << _counter << " - " << text - << "\n"; + << '\n'; } /////////////////////////////////////////////////////////////////////////////// @@ -452,12 +452,13 @@ void UnitTest::skip (const std::string& text) { ++_counter; ++_skipped; - std::cout << yellow ("skip") + std::cout << yellow ("ok") << " " << _counter << " - " << text - << "\n"; + << " # skip" + << '\n'; } ///////////////////////////////////////////////////////////////////////////////