TW-1936: Tweak tests to have fuller TAP compliance

- Thanks to Paul J. Fenwick.

ChangeLog

@@ -10,6 +10,8 @@
           (thanks to Eric Hymowitz, Paul Fenwick).
 - TS-34   Tasksh throw a warning at the end of a review command
           (thanks to bjonnh).
+- TW-1936 Tweak tests to have fuller TAP compliance
+          (thanks to Paul J. Fenwick)
 - Review report now defaults to 6 days instead of 1 weeķ, which is more
   convenient for those who review weekly
           (thanks to Dirk Deimeke).

test/problems

@@ -54,6 +54,15 @@ if __name__ == "__main__":
 
     file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$")
     timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$")
+
+    expected_fail = re.compile(r"^not ok.*?#\s*TODO", re.I)
+    unexpected_pass = re.compile(r"^ok .*?#\s*TODO", re.I)
+    skip = re.compile(r"^ok .*?#\s*skip", re.I)
+    ok = re.compile(r"^ok ", re.I)
+    not_ok = re.compile(r"^not ok", re.I)
+    comment = re.compile(r"^#")
+    plan = re.compile(r"^1..\d+\s*(?:#.*)?$")
+
     start = None
     stop = None
 
@@ -68,31 +77,37 @@ if __name__ == "__main__":
             if match:
                 filename = match.group(1)
 
-            if line.startswith("ok "):
+            elif expected_fail.match(line):
-                passed[filename] += 1
-
-            if line.startswith("not "):
-                errors[filename] += 1
-
-            if line.startswith("skip "):
-                skipped[filename] += 1
-
-            if line.startswith("# EXPECTED_FAILURE:"):
                 expected[filename] += 1
 
-            if line.startswith("# UNEXPECTED_SUCCESS:"):
+            elif unexpected_pass.match(line):
                 unexpected[filename] += 1
 
+            elif skip.match(line):
+                skipped[filename] += 1
+
+            # It's important these come last, since they're subpatterns of the above
+
+            elif ok.match(line):
+                passed[filename] += 1
+
+            elif not_ok.match(line):
+                errors[filename] += 1
+
+            elif comment.match(line):
+                pass
+
+            elif plan.match(line):
+                pass
+
+            else:
+                # Uncomment if you want to see malformed things we caught as well...
+                # print(color("Malformed TAP (" + filename + "): " + line, "red"))
+                pass
+
         # Last line contains the ending timestamp
         stop = float(timestamp.match(line).group(1))
 
-    # Remove expected failures from the skipped tests category
-    for filename, value in expected.items():
-        if skipped[filename] == value:
-            del skipped[filename]
-        else:
-            skipped[filename] -= value
-
     v = "{0:>5d}"
     passed_str = "Passed:" + pad(24)
     passed_int = v.format(sum(passed.values()))

test/simpletap/__init__.py

@@ -1,4 +1,5 @@
 ###############################################################################
+# taskwarrior - a command line task list manager.
 #
 # Copyright 2006 - 2017, Paul Beckingham, Federico Hernandez.
 #
@@ -154,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult):
         if status:
 
             if status == "SKIP":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
+                self.stream.writeln("{0} {1} - {2}: {3} # skip".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                    color("ok", "yellow"), self.testsRun, filename, desc)
                 )
             elif status == "EXPECTED_FAILURE":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
+                self.stream.writeln("{0} {1} - {2}: {3} # TODO".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                    color("not ok", "yellow"), self.testsRun, filename, desc)
                 )
             else:
                 self.stream.writeln("{0} {1} - {2}: {3}".format(
@@ -226,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner):
         unittest.signals.registerResult(result)
         result.failfast = self.failfast
 
+        # TAP requires output is on STDOUT.
+        # TODO: Define this at __init__ time
+        result.stream = unittest.runner._WritelnDecorator(sys.stdout)
+
         with warnings.catch_warnings():
             if getattr(self, "warnings", None):
                 # if self.warnings is set, use it to filter all the warnings

test/test.cpp

@@ -51,7 +51,7 @@ UnitTest::UnitTest (int planned)
 , _failed (0)
 , _skipped (0)
 {
-  std::cout << "1.." << _planned << "\n";
+  std::cout << "1.." << _planned << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -99,14 +99,14 @@ void UnitTest::plan (int planned)
   _failed = 0;
   _skipped = 0;
 
-  std::cout << "1.." << _planned << "\n";
+  std::cout << "1.." << _planned << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 void UnitTest::planMore (int extra)
 {
   _planned += extra;
-  std::cout << "1.." << _planned << "\n";
+  std::cout << "1.." << _planned << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -122,7 +122,7 @@ void UnitTest::ok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -132,7 +132,7 @@ void UnitTest::ok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
 }
 
@@ -149,7 +149,7 @@ void UnitTest::notok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -159,7 +159,7 @@ void UnitTest::notok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
 }
 
@@ -175,7 +175,7 @@ void UnitTest::is (bool actual, bool expected, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -189,7 +189,7 @@ void UnitTest::is (bool actual, bool expected, const std::string& name)
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -205,7 +205,7 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -219,7 +219,7 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name)
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -235,7 +235,7 @@ void UnitTest::is (int actual, int expected, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -249,7 +249,7 @@ void UnitTest::is (int actual, int expected, const std::string& name)
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -265,7 +265,7 @@ void UnitTest::is (double actual, double expected, const std::string& name)
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -279,7 +279,7 @@ void UnitTest::is (double actual, double expected, const std::string& name)
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -295,7 +295,7 @@ void UnitTest::is (double actual, double expected, double tolerance, const std::
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -309,7 +309,7 @@ void UnitTest::is (double actual, double expected, double tolerance, const std::
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -325,7 +325,7 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -339,7 +339,7 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri
               << expected
               << "\n#      got: "
               << actual
-              << "\n";
+              << '\n';
   }
 }
 
@@ -358,7 +358,7 @@ void UnitTest::is (
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -392,7 +392,7 @@ void UnitTest::is (
               << _counter
               << " - "
               << name
-              << "\n";
+              << '\n';
   }
   else
   {
@@ -418,7 +418,7 @@ void UnitTest::diag (const std::string& text)
   auto end   = text.find_last_not_of  (" \t\n\r\f");
   if (start != std::string::npos &&
       end   != std::string::npos)
-    std::cout << "# " << text.substr (start, end - start + 1) << "\n";
+    std::cout << "# " << text.substr (start, end - start + 1) << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -431,7 +431,7 @@ void UnitTest::pass (const std::string& text)
             << _counter
             << " - "
             << text
-            << "\n";
+            << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -444,7 +444,7 @@ void UnitTest::fail (const std::string& text)
             << _counter
             << " - "
             << text
-            << "\n";
+            << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -452,12 +452,13 @@ void UnitTest::skip (const std::string& text)
 {
   ++_counter;
   ++_skipped;
-  std::cout << yellow ("skip")
+  std::cout << yellow ("ok")
             << " "
             << _counter
             << " - "
             << text
-            << "\n";
+            << " # skip"
+            << '\n';
 }
 
 ///////////////////////////////////////////////////////////////////////////////