From efc56d3ab6e676af17f8414ef8f1dc70649758c3 Mon Sep 17 00:00:00 2001
From: Tomas Babej <tomas@tbabej.com>
Date: Sat, 5 Dec 2020 21:19:27 -0500
Subject: [PATCH] tests: Implement mechanism for expected failures in C++ tests

Implements detection of unexpected successes and expected failures. Both
classes are represented in the TAP output as 'not ok', unexpected
successes with '# FIXED' metadata and expected failures as '# TODO'.

This brings C++ tests to feature parity with Python-based ones when it
comes to expected failures and unexpected successes.
---
 test/test.cpp | 72 +++++++++++++++++++++++++++++++++++++--------------
 test/test.h   | 20 +++++++-------
 2 files changed, 62 insertions(+), 30 deletions(-)

diff --git a/test/test.cpp b/test/test.cpp
index 70491cf72..95f690d00 100644
--- a/test/test.cpp
+++ b/test/test.cpp
@@ -110,11 +110,13 @@ void UnitTest::planMore (int extra)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::ok (bool expression, const std::string& name)
+void UnitTest::ok (bool expression, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
 
-  if (expression)
+  bool success = expression;
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -132,16 +134,19 @@ void UnitTest::ok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << '\n';
   }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::notok (bool expression, const std::string& name)
+void UnitTest::notok (bool expression, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
 
-  if (!expression)
+  bool success = not expression;
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -159,15 +164,18 @@ void UnitTest::notok (bool expression, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << '\n';
   }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (bool actual, bool expected, const std::string& name)
+void UnitTest::is (bool actual, bool expected, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -185,6 +193,7 @@ void UnitTest::is (bool actual, bool expected, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -194,10 +203,12 @@ void UnitTest::is (bool actual, bool expected, const std::string& name)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (size_t actual, size_t expected, const std::string& name)
+void UnitTest::is (size_t actual, size_t expected, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -215,6 +226,7 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -224,10 +236,12 @@ void UnitTest::is (size_t actual, size_t expected, const std::string& name)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (int actual, int expected, const std::string& name)
+void UnitTest::is (int actual, int expected, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -245,6 +259,7 @@ void UnitTest::is (int actual, int expected, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -254,10 +269,12 @@ void UnitTest::is (int actual, int expected, const std::string& name)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (double actual, double expected, const std::string& name)
+void UnitTest::is (double actual, double expected, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -275,6 +292,7 @@ void UnitTest::is (double actual, double expected, const std::string& name)
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -284,10 +302,12 @@ void UnitTest::is (double actual, double expected, const std::string& name)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (double actual, double expected, double tolerance, const std::string& name)
+void UnitTest::is (double actual, double expected, double tolerance, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (fabs (actual - expected) <= tolerance)
+  bool success = (fabs (actual - expected) <= tolerance);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -305,6 +325,7 @@ void UnitTest::is (double actual, double expected, double tolerance, const std::
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -314,10 +335,12 @@ void UnitTest::is (double actual, double expected, double tolerance, const std::
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-void UnitTest::is (unsigned char actual, unsigned char expected, const std::string& name)
+void UnitTest::is (unsigned char actual, unsigned char expected, const std::string& name, bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -335,6 +358,7 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: "
               << expected
               << "\n#      got: "
@@ -347,10 +371,13 @@ void UnitTest::is (unsigned char actual, unsigned char expected, const std::stri
 void UnitTest::is (
   const std::string& actual,
   const std::string& expected,
-  const std::string& name)
+  const std::string& name,
+  bool expfail /* = false */)
 {
   ++_counter;
-  if (actual == expected)
+  bool success = (actual == expected);
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -368,6 +395,7 @@ void UnitTest::is (
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: '"
               << expected
               << "'"
@@ -381,10 +409,13 @@ void UnitTest::is (
 void UnitTest::is (
   const char* actual,
   const char* expected,
-  const std::string& name)
+  const std::string& name,
+  bool expfail /* = false */)
 {
   ++_counter;
-  if (! strcmp (actual, expected))
+  bool success = (! strcmp (actual, expected));
+
+  if (success and ! expfail)
   {
     ++_passed;
     std::cout << green ("ok")
@@ -402,6 +433,7 @@ void UnitTest::is (
               << _counter
               << " - "
               << name
+              << (expfail ? (success ? " # FIXED" : " # TODO") : "")
               << "\n# expected: '"
               << expected
               << "'"
diff --git a/test/test.h b/test/test.h
index 411d68534..1e2b49549 100644
--- a/test/test.h
+++ b/test/test.h
@@ -38,16 +38,16 @@ public:
 
   void plan (int);
   void planMore (int);
-  void ok (bool, const std::string&);
-  void notok (bool, const std::string&);
-  void is (bool, bool, const std::string&);
-  void is (size_t, size_t, const std::string&);
-  void is (int, int, const std::string&);
-  void is (double, double, const std::string&);
-  void is (double, double, double, const std::string&);
-  void is (unsigned char, unsigned char, const std::string&);
-  void is (const std::string&, const std::string&, const std::string&);
-  void is (const char*, const char*, const std::string&);
+  void ok (bool, const std::string&, bool expfail = false);
+  void notok (bool, const std::string&, bool expfail = false);
+  void is (bool, bool, const std::string&, bool expfail = false);
+  void is (size_t, size_t, const std::string&, bool expfail = false);
+  void is (int, int, const std::string&, bool expfail = false);
+  void is (double, double, const std::string&, bool expfail = false);
+  void is (double, double, double, const std::string&, bool expfail = false);
+  void is (unsigned char, unsigned char, const std::string&, bool expfail = false);
+  void is (const std::string&, const std::string&, const std::string&, bool expfail = false);
+  void is (const char*, const char*, const std::string&, bool expfail = false);
   void diag (const std::string&);
   void pass (const std::string&);
   void fail (const std::string&);