add ctest as test driver (#3446)

.github/workflows/tests.yaml

@@ -99,10 +99,10 @@ jobs:
           GITHUB_USER: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CONTAINER: ${{ matrix.dockerfile }}
-        run: docker-compose build test-$CONTAINER
+        run: docker-compose build test-${{ env.CONTAINER }}
 
       - name: Test ${{ matrix.name }}
-        run: docker-compose run test-$CONTAINER
+        run: docker-compose run test-${{ env.CONTAINER }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CONTAINER: ${{ matrix.dockerfile }}

CMakeLists.txt

@@ -1,4 +1,5 @@
 cmake_minimum_required (VERSION 3.22)
+enable_testing()
 
 set (CMAKE_EXPORT_COMPILE_COMMANDS ON)
 

doc/devel/contrib/development.md

@@ -2,7 +2,7 @@
 
 ## Satisfy the Requirements:
 
- * CMake 3.0 or later
+ * CMake 3.22 or later
  * gcc 7.0 or later, clang 6.0 or later, or a compiler with full C++17 support
  * libuuid (if not on macOS)
  * Rust 1.64.0 or higher (hint: use https://rustup.rs/ instead of using your system's package manager)
@@ -49,28 +49,47 @@ cmake --build build-clang
 ```
 
 ## Run the Test Suite:
-First switch to the test directory:
+For running the test suite [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) is used.
+Before one can run the test suite the `task_executable` must be built.
+After that also the `build_tests` target must be build, which can be done over:
+```sh
+cmake --build build --target build_tests
+```
+Again you may also use the `-j <number-of-jobs>` option for parallel builds.
 
+Now one can invoke `ctest` to run the tests.
+```sh
+ctest --test-dir build
 ```
-    $ cd build/test
+This would run all the test in serial and might take some time.
+
+### Running tests in parallel
+```sh
+ctest --test-dir build -j <number-of-jobs>
 ```
-Then you can run all tests, showing details, with
+
+Further it is adviced to add the `--output-on-failure` option to `ctest`, to recieve a verbose output if a test is failing as well as the `--rerun-failed` flag, to invoke in subsequent runs only the failed ones.
+
+### Running specific tests
+For this case one can use the `-R <regex>` or `--tests-regex <regex>`  option to run only the tests matching the regular expression.
+Running only the `cpp` tests can then be achieved over
+```sh
+ctest --test-dir build -R cpp
 ```
-    $ make VERBOSE=1 test
+or running the `variant_*` tests
+```sh
+ctest --test-dir build -R variant
 ```
-Alternately, run the tests with the details hidden in `all.log`:
+
-```
+### Repeating a test case
-    $ ./run_all
+In order to find sporadic test failures the `--repeat` flag can be used.
-```
+```sh
-Either way, you can get a summary of any test failures with:
+ctest --test-dir build -R cpp --repeat-until-fail 10
-```
-    $ ./problems
-```
-You can run a single test suite, with source file `foo.test.cpp` or `foo.test.py`, with
-```
-    $ make foo.test
 ```
 
+There are more options to `ctest` such as `--progress`, allowing to have a less verbose output.
+They can be found in the [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) man page.
+
 Note that any development should be performed using a git clone, and the current development branch.
 The source tarballs do not reflect HEAD, and do not contain the test suite.
 Follow the [GitHub flow](https://docs.github.com/en/get-started/quickstart/github-flow) for creating a pull request.

test/CMakeLists.txt

@@ -1,13 +1,5 @@
 cmake_minimum_required (VERSION 3.22)
 
-# This is a work-around for the following CMake issue:
-# https://gitlab.kitware.com/cmake/cmake/issues/16062
-# The issue has been fixed in CMake 3.11.0; the policy is set
-# to OLD for compatibility with older versions of CMake only.
-if(POLICY CMP0037 AND ${CMAKE_VERSION} VERSION_LESS "3.11.0")
-  cmake_policy(SET CMP0037 OLD)
-endif()
-
 include_directories (${CMAKE_SOURCE_DIR}
                      ${CMAKE_SOURCE_DIR}/src
                      ${CMAKE_SOURCE_DIR}/src/tc/lib
@@ -51,30 +43,23 @@ set (test_SRCS
   view.test.cpp
   )
 
-add_custom_target (test ./run_all --verbose
-                        DEPENDS ${test_SRCS} task_executable
-                        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
-
 add_custom_target (build_tests DEPENDS ${test_SRCS}
                                WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/test)
 
 foreach (src_FILE ${test_SRCS})
   add_executable (${src_FILE} ${src_FILE} test.cpp)
   target_link_libraries (${src_FILE} task tc commands columns libshared task tc commands columns libshared task commands columns libshared ${TASK_LIBRARIES})
+  add_dependencies (${src_FILE} task_executable)
   if (DARWIN)
     target_link_libraries (${src_FILE} "-framework CoreFoundation -framework Security -framework SystemConfiguration")
   endif (DARWIN)
 
-  # Add a custom `foo.test` target.
+  add_test(NAME ${src_FILE}
-  string(REGEX REPLACE "\\.[^.]*$" "" test_target ${src_FILE})
+    COMMAND ${src_FILE}
-  add_custom_target (${test_target}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                     ./${src_FILE}
+    )
-                     DEPENDS ${src_FILE}
-                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
 endforeach (src_FILE)
 
-configure_file(run_all run_all COPYONLY)
-configure_file(problems problems COPYONLY)
 configure_file(bash_tap.sh bash_tap.sh COPYONLY)
 configure_file(bash_tap_tw.sh bash_tap_tw.sh COPYONLY)
 
@@ -205,15 +190,8 @@ set (pythonTests
 foreach (python_Test ${pythonTests})
   configure_file(${python_Test} ${python_Test} COPYONLY)
 
-  # Add a custom `foo.test` target.
+  add_test(NAME ${python_Test}
-  string(REGEX REPLACE "\\.[^.]*$" "" test_target ${python_Test})
+    COMMAND ${Python_EXECUTABLE} ${python_Test}
-  add_custom_target (${test_target}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                     ./${python_Test}
+    )
-                     DEPENDS ${python_Test}
-                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
 endforeach(python_Test)
-
-#SET(CMAKE_BUILD_TYPE gcov)
-#SET(CMAKE_CXX_FLAGS_GCOV "--coverage")
-#SET(CMAKE_C_FLAGS_GCOV "--coverage")
-#SET(CMAKE_EXE_LINKER_FLAGS_GCOV "--coverage")

test/README.md

@@ -1,36 +1,18 @@
-README
+## Running Tests
-======
-
-This is the task.git/test/README file, and contains notes about the Taskwarrior
-test suite.
-
-
-Running Tests
--------------
-
 Do this to run all tests:
+```shell
+cmake --build build --target build_tests
+ctest --test-dir build
+```
 
-  $ cd test && make && ./run_all && ./problems
+All tests produce TAP (Test Anything Protocol) output.
+In order to run the tests in parallel add the `--parallel <# of threads>` or shortly `-j <# of threads>` option to `ctest`.
+Depending on your IDE, all tests might also be available under the `All CTest` target.
+Keep in mind that the tests are not automatically rebuild if a source file is changes, it requires a manual rebuild.
 
-All unit tests produce TAP (Test Anything Protocol) output, and are run by the
+Please also have a look at [development.md](../doc/devel/contrib/development.md) for more information on how to run tests as well as some information about `ctest`.
-'run_all' test harness.
 
-The 'run_all' script produces an 'all.log' file which is the accumulated output
+## Architecture
-of all tests. Before executing 'run_all' you need to compile the C++ unit
-tests, by running 'make' in the 'test' directory.
-
-The script 'problems' will list all the tests that fail, with a count of the
-failing tests, once you have run all the tests and produced an 'all.log' file.
-
-Any TAP harness may be used.
-
-Note that adding the '--serial' option to ./run_all, all tests are executed
-serially.  The default runs Python, C++ and Bash tests in parallel. Using
-'--serial' will make for a slower test run.
-
-
-Architecture
-------------
 
 There are three varieties of tests:
 
@@ -45,17 +27,18 @@ There are three varieties of tests:
     tests are small, quick tests, not intended to be permanent.
 
 All tests are named with the pattern '*.test.py', '*.test.sh', or '*.test.cpp',
-and any other forms are not run by the test harness. Additionally a test must
+and any other forms are not run by the test harness.
-be set executable (chmod +x) for it to be run. In the case of Python tests one
+In the case of Python tests one can still run them manually by launching them with 'python testname.test.py' or simply './testname.test.py'.
-can still run them manually by launching them with 'python testname.test.py' or
+
-simply './testname.test.py'. It also allows us to keep tests submitted for bugs
+If a test is failing and can not be fixed, it can be marked as `WILL_FAIL` in the `CMakeLists.txt` file.
-that are not scheduled to be fixed in the upcoming release, and we don't want
+See the [WILL_FAIL](https://cmake.org/cmake/help/latest/prop_test/WILL_FAIL.html) documentation for more information.
-the failing tests to prevent us from seeing 100% pass rate for the bugs we
+However, please keep in mind that such tests should be fixed as soon as possible as well as proper documentation should be added to the issue tracker.
-*have* fixed.
+
+It also allows us to keep tests submitted for bugs that are not scheduled to be fixed in the upcoming release, and we don't want
+the failing tests to prevent us from seeing 100% pass rate for the bugs we *have* fixed.
 
 
-Goals
+## Goals
------
 
 The test suite is evolving, and becoming a better tool for determining whether
 code is ready for release. There are goals that shape these changes, and they
@@ -71,16 +54,14 @@ are:
     There is simply no point in testing a feature twice, in the same manner.
 
 
-What Makes a Good Test
+## What Makes a Good Test
-----------------------
 
 A good test ensures that a feature is functioning as expected, and contains
 both positive and negative aspects, or in other words looks for expected
 behavior as well as looking for the absence of unexpected behavior.
 
 
-Conventions for writing a test
+## Conventions for writing a test
-------------------------------
 
 If you wish to contribute tests, please consider the following guidelines:
 
@@ -109,14 +90,12 @@ If you wish to contribute tests, please consider the following guidelines:
     a live test that is skipped, than no test.
 
 
-How to Submit a Test Change/Addition
+## How to Submit a Test Change/Addition
-------------------------------------
 
 Mail it to support@gothenburgbitfactory.org, or attach it to an open bug.
 
 
-Wisdom
+## Wisdom
-------
 
 Here are some guildelines that may help:
 
@@ -146,8 +125,7 @@ Here are some guildelines that may help:
     are reported.
 
 
-TODO
+## TODO
-----
 
 For anyone looking for test-related tasks to take on, here are some suggestions:
 
@@ -161,6 +139,4 @@ For anyone looking for test-related tasks to take on, here are some suggestions:
 
   * All the attribute modifiers need to be tested, only a few are.
 
-  * Aliases are not well tested, and fragile.
+  * Aliases are not well tested, and fragile.
-
----

test/docker/arch

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test/
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/debianstable

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/debiantesting

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/fedora39

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/fedora40

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/opensuse

@@ -25,7 +25,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/ubuntu2004

@@ -34,7 +34,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/docker/ubuntu2204

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
+RUN cmake --build build --target build_tests -j 8
-RUN make -j8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]

test/problems

@@ -1,149 +0,0 @@
-#!/usr/bin/env python3
-
-from __future__ import print_function
-import sys
-import re
-import argparse
-from collections import defaultdict
-
-
-def color(text, c):
-    """
-    Add color on the keyword that identifies the state of the test
-    """
-    if sys.stdout.isatty():
-        clear = "\033[0m"
-
-        colors = {
-            "red": "\033[1m\033[91m",
-            "yellow": "\033[1m\033[93m",
-            "green": "\033[1m\033[92m",
-        }
-        return colors[c] + text + clear
-    else:
-        return text
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Report on test results")
-    parser.add_argument('--summary', action="store_true",
-                        help="Display only the totals in each category")
-    parser.add_argument('tapfile', default="all.log", nargs="?",
-                        help="File containing TAP output")
-    return parser.parse_args()
-
-
-def print_category(tests):
-    if not cmd_args.summary:
-        for key in sorted(tests):
-            print("%-32s %4d" % (key, tests[key]))
-
-
-def pad(i):
-    return " " * i
-
-
-if __name__ == "__main__":
-    cmd_args = parse_args()
-
-    errors = defaultdict(int)
-    skipped = defaultdict(int)
-    expected = defaultdict(int)
-    unexpected = defaultdict(int)
-    passed = defaultdict(int)
-
-    file = re.compile(r"^# (?:./)?(\S+\.test)(?:\.py|\.cpp)?$")
-    timestamp = re.compile(r"^# (\d+(?:\.\d+)?) ==>.*$")
-
-    expected_fail = re.compile(r"^not ok.*?#\s*TODO", re.I)
-    unexpected_pass = re.compile(r"^not ok .*?#\s*FIXED", re.I)
-    skip = re.compile(r"^ok .*?#\s*skip", re.I)
-    ok = re.compile(r"^ok ", re.I)
-    not_ok = re.compile(r"^not ok", re.I)
-    comment = re.compile(r"^#")
-    plan = re.compile(r"^1..\d+\s*(?:#.*)?$")
-
-    start = None
-    stop = None
-
-    with open(cmd_args.tapfile) as fh:
-        for line in fh:
-            if start is None:
-                # First line contains the starting timestamp
-                start = float(timestamp.match(line).group(1))
-                continue
-
-            match = file.match(line)
-            if match:
-                filename = match.group(1)
-
-            elif expected_fail.match(line):
-                expected[filename] += 1
-
-            elif unexpected_pass.match(line):
-                unexpected[filename] += 1
-
-            elif skip.match(line):
-                skipped[filename] += 1
-
-            # It's important these come last, since they're subpatterns of the above
-
-            elif ok.match(line):
-                passed[filename] += 1
-
-            elif not_ok.match(line):
-                errors[filename] += 1
-
-            elif comment.match(line):
-                pass
-
-            elif plan.match(line):
-                pass
-
-            else:
-                # Uncomment if you want to see malformed things we caught as well...
-                # print(color("Malformed TAP (" + filename + "): " + line, "red"))
-                pass
-
-        # Last line contains the ending timestamp
-        stop = float(timestamp.match(line).group(1))
-
-    v = "{0:>5d}"
-    passed_str = "Passed:" + pad(24)
-    passed_int = v.format(sum(passed.values()))
-    error_str = "Failed:" + pad(24)
-    error_int = v.format(sum(errors.values()))
-    unexpected_str = "Unexpected successes:" + pad(10)
-    unexpected_int = v.format(sum(unexpected.values()))
-    skipped_str = "Skipped:" + pad(23)
-    skipped_int = v.format(sum(skipped.values()))
-    expected_str = "Expected failures:" + pad(13)
-    expected_int = v.format(sum(expected.values()))
-    runtime_str = "Runtime:" + pad(20)
-    runtime_int = "{0:>8.2f} seconds".format(stop - start)
-    details_str = "For details run 'make problems'"
-
-    if cmd_args.summary:
-        print(color(passed_str, "green"), passed_int)
-        print(color(error_str, "red"), error_int)
-        print(color(unexpected_str, "red"), unexpected_int)
-        print(color(skipped_str, "yellow"), skipped_int)
-        print(color(expected_str, "yellow"), expected_int)
-        print(runtime_str, runtime_int)
-        print(details_str)
-
-    else:
-        print(color(error_str, "red"))
-        print_category(errors)
-        print()
-        print(color(unexpected_str, "red"))
-        print_category(unexpected)
-        print()
-        print(color(skipped_str, "yellow"))
-        print_category(skipped)
-        print()
-        print(color(expected_str, "yellow"))
-        print_category(expected)
-
-    # If we encoutered any failures, return non-zero code
-    sys.exit(1 if int(error_int) or int(unexpected_int) else 0)

test/run_all

@@ -1,242 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function
-import os
-import sys
-import glob
-import argparse
-import logging
-import time
-from multiprocessing import cpu_count
-from threading import Thread
-from subprocess import call, Popen, PIPE
-
-if sys.version_info > (3,):
-    import codecs
-
-try:
-    # python 2
-    from Queue import Queue, Empty
-except ImportError:
-    # python 3
-    from queue import Queue, Empty
-
-TIMEOUT = .2
-
-
-def run_test(testqueue, outqueue, threadname):
-    start = time.time()
-    while True:
-        try:
-            test = testqueue.get(block=True, timeout=TIMEOUT)
-        except Empty:
-            break
-
-        log.info("Running test %s", test)
-
-        try:
-            p = Popen(os.path.abspath(test), stdout=PIPE, stderr=PIPE,
-                      env=os.environ)
-            out, err = p.communicate()
-        except Exception as e:
-            log.exception(e)
-            # Premature end
-            break
-
-        if sys.version_info > (3,):
-            out, err = out.decode('utf-8'), err.decode('utf-8')
-
-        if p.returncode != 0:
-            out = out + "\nnot ok - test executable failed\n"
-
-        output = ("# {0}\n".format(os.path.basename(test)), out, err)
-        log.debug("Collected output %s", output)
-        outqueue.put(output)
-
-        testqueue.task_done()
-
-    log.warning("Finished %s thread after %s seconds",
-                threadname, round(time.time() - start, 3))
-
-
-class TestRunner(object):
-    def __init__(self):
-        self.threads = []
-        if sys.version_info > (3,):
-            self.tap = open(cmd_args.tapfile, 'w', errors='ignore')
-        else:
-            self.tap = open(cmd_args.tapfile, 'w')
-
-        self._parallelq = Queue()
-        self._serialq = Queue()
-        self._outputq = Queue()
-
-    def _find_tests(self):
-        for test in glob.glob("*.test.sh") + glob.glob("*.test.py") + glob.glob("*.test.cpp"):
-            if os.access(test, os.X_OK):
-                # Executables only
-                if self._is_parallelizable(test):
-                    log.debug("Treating as parallel: %s", test)
-                    self._parallelq.put(test)
-                else:
-                    log.debug("Treating as serial: %s", test)
-                    self._serialq.put(test)
-            else:
-                log.debug("Ignored test %s as it is not executable", test)
-
-        log.info("Parallel tests: %s", self._parallelq.qsize())
-        log.info("Serial tests: %s", self._serialq.qsize())
-
-    def _prepare_threads(self):
-        # Serial thread
-        self.threads.append(
-            Thread(target=run_test, args=(self._serialq, self._outputq, "Serial"))
-        )
-        # Parallel threads
-        self.threads.extend([
-            Thread(target=run_test, args=(self._parallelq, self._outputq, "Parallel"))
-            for i in range(cpu_count())
-        ])
-        log.info("Spawned %s threads to run tests", len(self.threads))
-
-    def _start_threads(self):
-        for thread in self.threads:
-            # Threads die when main thread dies
-            log.debug("Starting thread %s", thread)
-            thread.daemon = True
-            thread.start()
-
-    def _print_timestamp_to_tap(self):
-        now = time.time()
-        timestamp = "# {0} ==> {1}\n".format(
-            now,
-            time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now)),
-        )
-
-        log.debug("Adding timestamp %s to TAP file", timestamp)
-        self.tap.write(timestamp)
-
-    def _is_parallelizable(self, test):
-        if cmd_args.serial:
-            return False
-
-        # This is a pretty weird way to do it, and not realiable.
-        # We are dealing with some binary tests though.
-        with open(test, 'rb') as fh:
-            header = fh.read(100).split(b"\n")
-            if len(header) >= 2 and \
-                ((b"!#/usr/bin/env python3" in header[0]) or \
-                (header[1][-14:] == b"bash_tap_tw.sh")):
-                return True
-            else:
-                return False
-
-    def _get_remaining_tests(self):
-        return self._parallelq.qsize() + self._serialq.qsize()
-
-    def is_running(self):
-        for thread in self.threads:
-            if thread.is_alive():
-                return True
-
-        return False
-
-    def start(self):
-        self._find_tests()
-        self._prepare_threads()
-
-        self._print_timestamp_to_tap()
-
-        finished = 0
-        total = self._get_remaining_tests()
-
-        self._start_threads()
-
-        while self.is_running() or not self._outputq.empty():
-            try:
-                outputs = self._outputq.get(block=True, timeout=TIMEOUT)
-            except Empty:
-                continue
-
-            log.debug("Outputting to TAP: %s", outputs)
-
-            for output in outputs:
-                self.tap.write(output)
-
-                if cmd_args.verbose:
-                    sys.stdout.write(output)
-
-            self._outputq.task_done()
-            finished += 1
-
-            log.warning("Finished %s out of %s tests", finished, total)
-
-        self._print_timestamp_to_tap()
-
-        if not self._parallelq.empty() or not self._serialq.empty():
-            raise RuntimeError(
-                "Something went wrong, not all tests were ran. {0} "
-                "remaining.".format(self._get_remaining_tests()))
-
-    def show_report(self):
-        self.tap.flush()
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-        log.debug("Calling 'problems --summary' for report")
-        return call([os.path.abspath("problems"), "--summary", cmd_args.tapfile])
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Run Taskwarrior tests")
-    parser.add_argument('--verbose', '-v', action="store_true",
-                        help="Also send TAP output to stdout")
-    parser.add_argument('--logging', '-l', action="count",
-                        default=0,
-                        help="Logging level. -lll is the highest level")
-    parser.add_argument('--serial', action="store_true",
-                        help="Do not run tests in parallel")
-    parser.add_argument('--tapfile', default="all.log",
-                        help="File to use for TAP output")
-    return parser.parse_args()
-
-
-def main():
-    if sys.version_info > (3,):
-        sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
-
-    runner = TestRunner()
-    runner.start()
-
-    # Propagate the return code
-    return runner.show_report()
-
-
-if __name__ == "__main__":
-    cmd_args = parse_args()
-
-    if cmd_args.logging == 1:
-        level = logging.WARN
-    elif cmd_args.logging == 2:
-        level = logging.INFO
-    elif cmd_args.logging >= 3:
-        level = logging.DEBUG
-    else:
-        level = logging.ERROR
-
-    logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(message)s",
-        level=level,
-    )
-    log = logging.getLogger(__name__)
-
-    log.debug("Parsed commandline arguments: %s", cmd_args)
-
-    try:
-        sys.exit(main())
-    except Exception as e:
-        log.exception(e)
-        sys.exit(1)
-
-# vim: ai sts=4 et sw=4