Improve TAP compliance in test suite

This allows tools like `prove` to operate on our test-suite, allowing us
to do things like run recently-failing tests first, randomising test
order, and fancier reporting options.

Changes made:

- TAP output always goes to STDOUT, never to STDERR
- Skipped tests signaled as `ok 1 # skip` as per standard
- Expected failures signaled as `not ok 1 # TODO` as per standard
- `.prove` added to .gitignore
This commit is contained in:
Paul Fenwick 2017-11-24 03:50:52 -05:00 committed by Paul Beckingham
parent b3192b94b4
commit 3b46553f47
3 changed files with 11 additions and 5 deletions

1
.gitignore vendored
View file

@ -19,3 +19,4 @@ CPackSourceConfig.cmake
patches
*.exe
tutorials
.prove

View file

@ -155,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult):
if status:
if status == "SKIP":
self.stream.writeln("{0} {1} - {2}: {3}".format(
color("skip", "yellow"), self.testsRun, filename, desc)
self.stream.writeln("{0} {1} - {2}: {3} # skip".format(
color("ok", "yellow"), self.testsRun, filename, desc)
)
elif status == "EXPECTED_FAILURE":
self.stream.writeln("{0} {1} - {2}: {3}".format(
color("skip", "yellow"), self.testsRun, filename, desc)
self.stream.writeln("{0} {1} - {2}: {3} # TODO".format(
color("not ok", "yellow"), self.testsRun, filename, desc)
)
else:
self.stream.writeln("{0} {1} - {2}: {3}".format(
@ -227,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner):
unittest.signals.registerResult(result)
result.failfast = self.failfast
# TAP requires output is on STDOUT.
# TODO: Define this at __init__ time
result.stream = unittest.runner._WritelnDecorator(sys.stdout)
with warnings.catch_warnings():
if getattr(self, "warnings", None):
# if self.warnings is set, use it to filter all the warnings

View file

@ -452,11 +452,12 @@ void UnitTest::skip (const std::string& text)
{
++_counter;
++_skipped;
std::cout << yellow ("skip")
std::cout << yellow ("ok")
<< " "
<< _counter
<< " - "
<< text
<< " # skip"
<< "\n";
}