TW-1936: Tweak tests to have fuller TAP compliance

- Thanks to Paul J. Fenwick.

AUTHORS

@@ -25,6 +25,7 @@ The following submitted code, packages or analysis, and deserve special thanks:
   A M
   asmyers
   Lukas Barth
+  Paul J. Fenwick
 
 Thanks to the following, who submitted detailed bug reports and excellent
 suggestions:

ChangeLog

@@ -4,6 +4,8 @@
           (Thanks to Jörg Krause, Ben Boeckel).
 - TW-1845 Cygwin build fails, missing get_current_dir_name
           (thanks to hosaka).
+- TW-1936 Tweak tests to have fuller TAP compliance
+          (thanks to Paul J. Fenwick)
 - TI-27   Continue tracking by ID
           (thanks to Dennis Schubert)
 - TI-29   timew config can't add new value

test/problems

@@ -54,6 +54,15 @@ if __name__ == "__main__":
 
     file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$")
     timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$")
+
+    expected_fail = re.compile(r"^not ok.*?#\s*TODO", re.I)
+    unexpected_pass = re.compile(r"^ok .*?#\s*TODO", re.I)
+    skip = re.compile(r"^ok .*?#\s*skip", re.I)
+    ok = re.compile(r"^ok ", re.I)
+    not_ok = re.compile(r"^not ok", re.I)
+    comment = re.compile(r"^#")
+    plan = re.compile(r"^1..\d+\s*(?:#.*)?$")
+
     start = None
     stop = None
 
@@ -68,31 +77,37 @@ if __name__ == "__main__":
             if match:
                 filename = match.group(1)
 
-            if line.startswith("ok "):
+            elif expected_fail.match(line):
-                passed[filename] += 1
-
-            if line.startswith("not "):
-                errors[filename] += 1
-
-            if line.startswith("skip "):
-                skipped[filename] += 1
-
-            if line.startswith("# EXPECTED_FAILURE:"):
                 expected[filename] += 1
 
-            if line.startswith("# UNEXPECTED_SUCCESS:"):
+            elif unexpected_pass.match(line):
                 unexpected[filename] += 1
 
+            elif skip.match(line):
+                skipped[filename] += 1
+
+            # It's important these come last, since they're subpatterns of the above
+
+            elif ok.match(line):
+                passed[filename] += 1
+
+            elif not_ok.match(line):
+                errors[filename] += 1
+
+            elif comment.match(line):
+                pass
+
+            elif plan.match(line):
+                pass
+
+            else:
+                # Uncomment if you want to see malformed things we caught as well...
+                # print(color("Malformed TAP (" + filename + "): " + line, "red"))
+                pass
+
         # Last line contains the ending timestamp
         stop = float(timestamp.match(line).group(1))
 
-    # Remove expected failures from the skipped tests category
-    for filename, value in expected.items():
-        if skipped[filename] == value:
-            del skipped[filename]
-        else:
-            skipped[filename] -= value
-
     v = "{0:>5d}"
     passed_str = "Passed:" + pad(24)
     passed_int = v.format(sum(passed.values()))

test/simpletap/__init__.py

@@ -1,4 +1,5 @@
 ###############################################################################
+# taskwarrior - a command line task list manager.
 #
 # Copyright 2006 - 2017, Paul Beckingham, Federico Hernandez.
 #
@@ -154,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult):
         if status:
 
             if status == "SKIP":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
+                self.stream.writeln("{0} {1} - {2}: {3} # skip".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                    color("ok", "yellow"), self.testsRun, filename, desc)
                 )
             elif status == "EXPECTED_FAILURE":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
+                self.stream.writeln("{0} {1} - {2}: {3} # TODO".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                    color("not ok", "yellow"), self.testsRun, filename, desc)
                 )
             else:
                 self.stream.writeln("{0} {1} - {2}: {3}".format(
@@ -226,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner):
         unittest.signals.registerResult(result)
         result.failfast = self.failfast
 
+        # TAP requires output is on STDOUT.
+        # TODO: Define this at __init__ time
+        result.stream = unittest.runner._WritelnDecorator(sys.stdout)
+
         with warnings.catch_warnings():
             if getattr(self, "warnings", None):
                 # if self.warnings is set, use it to filter all the warnings

test/test.cpp

@@ -452,11 +452,12 @@ void UnitTest::skip (const std::string& text)
 {
   ++_counter;
   ++_skipped;
-  std::cout << yellow ("skip")
+  std::cout << yellow ("ok")
             << " "
             << _counter
             << " - "
             << text
+            << " # skip"
             << '\n';
 }