From d354d13263a798639dc4d77b269dbcc60de482d2 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Mon, 14 Jul 2014 11:37:35 +0100 Subject: [PATCH 01/11] Unittest - Bring back diag() for generating TAP output * Still needs work to make sure output is displayed after the test status (not ok ...) and not before. --- test/basetest/__init__.py | 1 + test/basetest/testing.py | 15 +++++++++++++++ test/bug.1254.t | 4 ++-- test/bug.1267.t | 4 ++-- test/bug.360.t | 4 ++-- test/template.t | 4 ++-- test/tw-1300.t | 4 ++-- test/tw-1306.t | 4 ++-- test/tw-285.t | 4 ++-- test/version.t | 4 ++-- 10 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 test/basetest/testing.py diff --git a/test/basetest/__init__.py b/test/basetest/__init__.py index 612335ca9..5933ba5c8 100644 --- a/test/basetest/__init__.py +++ b/test/basetest/__init__.py @@ -2,5 +2,6 @@ from .task import Task from .taskd import Taskd +from .testing import TestCase # vim: ai sts=4 et sw=4 diff --git a/test/basetest/testing.py b/test/basetest/testing.py new file mode 100644 index 000000000..b8e03e9c7 --- /dev/null +++ b/test/basetest/testing.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +import unittest +import sys + + +class TestCase(unittest.TestCase): + def diag(self, out): + sys.stdout.write("# --- diag start ---\n") + for line in out.split("\n"): + sys.stdout.write("#" + line + "\n") + sys.stdout.write("# --- diag end ---\n") + + +# vim: ai sts=4 et sw=4 diff --git a/test/bug.1254.t b/test/bug.1254.t index 4ece13ff6..4ca59a8ab 100755 --- a/test/bug.1254.t +++ b/test/bug.1254.t @@ -33,10 +33,10 @@ import unittest # Ensure python finds the local simpletap and basetest modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class TestBug1254(unittest.TestCase): +class TestBug1254(TestCase): def setUp(self): self.t = Task() diff --git a/test/bug.1267.t b/test/bug.1267.t index b582337fc..d6f300ad8 100755 --- a/test/bug.1267.t +++ b/test/bug.1267.t @@ -32,10 +32,10 @@ import unittest # Ensure python finds the local simpletap and basetest modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class TestBug1267(unittest.TestCase): +class TestBug1267(TestCase): def setUp(self): self.t = Task() diff --git a/test/bug.360.t b/test/bug.360.t index cc9185241..1faa5d620 100755 --- a/test/bug.360.t +++ b/test/bug.360.t @@ -33,10 +33,10 @@ import unittest # Ensure python finds the local simpletap and basetest modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class BaseTestBug360(unittest.TestCase): +class BaseTestBug360(TestCase): def setUp(self): """Executed before each test in the class""" self.t = Task() diff --git a/test/template.t b/test/template.t index b050ef5d9..d9bea04ce 100755 --- a/test/template.t +++ b/test/template.t @@ -8,10 +8,10 @@ from datetime import datetime # Ensure python finds the local simpletap module sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task, Taskd +from basetest import Task, Taskd, TestCase -class TestCase(unittest.TestCase): +class TestBugNumber(TestCase): @classmethod def setUpClass(cls): """Executed once before any test in the class""" diff --git a/test/tw-1300.t b/test/tw-1300.t index 100645d7c..36ff52b2e 100755 --- a/test/tw-1300.t +++ b/test/tw-1300.t @@ -32,10 +32,10 @@ import unittest # Ensure python finds the local simpletap and basetest modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class TestBug1300(unittest.TestCase): +class TestBug1300(TestCase): @classmethod def setUp(cls): cls.t = Task() diff --git a/test/tw-1306.t b/test/tw-1306.t index 4574a1925..ad8b23873 100755 --- a/test/tw-1306.t +++ b/test/tw-1306.t @@ -31,10 +31,10 @@ import os import unittest sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class TestBug1306(unittest.TestCase): +class TestBug1306(TestCase): def setUp(self): self.t = Task() diff --git a/test/tw-285.t b/test/tw-285.t index 67ddc9251..f667dd68c 100755 --- a/test/tw-285.t +++ b/test/tw-285.t @@ -32,10 +32,10 @@ import unittest # Ensure python finds the local simpletap and basetest modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class Test285(unittest.TestCase): +class Test285(TestCase): @classmethod def setUpClass(cls): cls.t = Task() diff --git a/test/version.t b/test/version.t index e392e9eb9..5aa486f67 100755 --- a/test/version.t +++ b/test/version.t @@ -33,10 +33,10 @@ from datetime import datetime # Ensure python finds the local simpletap module sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from basetest import Task +from basetest import Task, TestCase -class TestVersion(unittest.TestCase): +class TestVersion(TestCase): def setUp(self): self.t = Task() From d419fb9560282a112016a705b471eb79095d5497 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Mon, 14 Jul 2014 11:44:28 +0100 Subject: [PATCH 02/11] Unittest - Don't escape new-line characters in TAP output --- test/simpletap/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/simpletap/__init__.py b/test/simpletap/__init__.py index 26ff3b108..df52f5a20 100644 --- a/test/simpletap/__init__.py +++ b/test/simpletap/__init__.py @@ -71,6 +71,8 @@ class TAPTestResult(unittest.result.TestResult): self.stream.writeln("# {0}: {1}".format(status, exception)) padding = " " * (len(status) + 3) for line in msg.splitlines(): + # Force displaying new-line characters as literal new lines + line = line.replace("\\n", "\n") self.stream.writeln("#{0}{1}".format(padding, line)) else: self.stream.writeln("ok {0} - {1}".format(self.testsRun, desc)) From 617183612cacf23b0dacdc732e45d03f68548957 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Mon, 14 Jul 2014 16:10:44 +0100 Subject: [PATCH 03/11] Unittest - Don't display "task diag" by default on error --- test/basetest/task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/basetest/task.py b/test/basetest/task.py index 6b77c9e7e..e7966ff67 100644 --- a/test/basetest/task.py +++ b/test/basetest/task.py @@ -127,7 +127,6 @@ class Task(object): merge_streams=merge_streams, env=self.env) if output[0] != 0: - output = self.diag(merge_streams_with=output) raise CommandError(command, *output) return output @@ -154,7 +153,6 @@ class Task(object): # output[0] is the exit code if output[0] == 0: - output = self.diag(merge_streams_with=output) raise CommandError(command, *output) return output From b77dfc631268c19398b6ed9bc9ba1f5cbab7a3a9 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 00:49:50 +0100 Subject: [PATCH 04/11] Unittest - Add which() backported from py3.3 --- test/basetest/utils.py | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/basetest/utils.py b/test/basetest/utils.py index a24e51a04..77fd1b21b 100644 --- a/test/basetest/utils.py +++ b/test/basetest/utils.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import sys import socket from subprocess import Popen, PIPE, STDOUT from .exceptions import CommandError @@ -95,4 +96,71 @@ def release_port(port): except KeyError: pass + +try: + from shutil import which +except ImportError: + # NOTE: This is shutil.which backported from python-3.3.3 + def which(cmd, mode=os.F_OK | os.X_OK, path=None): + """Given a command, mode, and a PATH string, return the path which + conforms to the given mode on the PATH, or None if there is no such + file. + + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result + of os.environ.get("PATH"), or can be overridden with a custom search + path. + + """ + # Check that a given file can be accessed with the correct mode. + # Additionally check that `file` is not a directory, as on Windows + # directories pass the os.access check. + def _access_check(fn, mode): + return (os.path.exists(fn) and os.access(fn, mode) + and not os.path.isdir(fn)) + + # If we're given a path with a directory part, look it up directly + # rather than referring to PATH directories. This includes checking + # relative to the current directory, e.g. ./script + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + return None + + if path is None: + path = os.environ.get("PATH", os.defpath) + if not path: + return None + path = path.split(os.pathsep) + + if sys.platform == "win32": + # The current directory takes precedence on Windows. + if os.curdir not in path: + path.insert(0, os.curdir) + + # PATHEXT is necessary to check on Windows. + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + # See if the given file matches any of the expected path + # extensions. This will allow us to short circuit when given + # "python.exe". If it does match, only test that one, otherwise we + # have to try others. + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + # On other platforms you don't have things like PATHEXT to tell you + # what file suffixes are executable, so just pass on cmd as-is. + files = [cmd] + + seen = set() + for dir in path: + normdir = os.path.normcase(dir) + if normdir not in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + return name + return None + # vim: ai sts=4 et sw=4 From 45c73fc4736bf78fcbf1bb32ef73c5959a017349 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 00:50:58 +0100 Subject: [PATCH 05/11] Unittest - Add code to check if taskd is available --- test/basetest/taskd.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/basetest/taskd.py b/test/basetest/taskd.py index 9b47b21fa..7616d7d39 100644 --- a/test/basetest/taskd.py +++ b/test/basetest/taskd.py @@ -7,7 +7,8 @@ import signal import atexit from time import sleep from subprocess import Popen -from .utils import find_unused_port, release_port, port_used, run_cmd_wait +from .utils import (find_unused_port, release_port, port_used, run_cmd_wait, + which) from .exceptions import CommandError try: @@ -33,7 +34,10 @@ class Taskd(object): A server can be stopped and started multiple times, but should not be started or stopped after being destroyed. """ - def __init__(self, taskd="taskd", certpath=None, address="127.0.0.1"): + DEFAULT_TASKD = "taskd" + + def __init__(self, taskd=DEFAULT_TASKD, certpath=None, + address="127.0.0.1"): """Initialize a Task server that runs in the background and stores data in a temporary folder @@ -260,4 +264,12 @@ class Taskd(object): raise AttributeError("Taskd instance has been destroyed. " "Create a new instance if you need a new server.") + @classmethod + def not_available(cls): + """Check if the taskd binary is available in the path""" + if which(cls.DEFAULT_TASKD): + return False + else: + return True + # vim: ai sts=4 et sw=4 From 5f50c44041c3812816d72fe63518d19cd8e1879b Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 00:51:29 +0100 Subject: [PATCH 06/11] Unittest - Add example of skipping taskd test on template.t --- test/template.t | 1 + 1 file changed, 1 insertion(+) diff --git a/test/template.t b/test/template.t index d9bea04ce..6bfb83a43 100755 --- a/test/template.t +++ b/test/template.t @@ -50,6 +50,7 @@ class TestBugNumber(TestCase): """Executed once after all tests in the class""" +@unittest.skipIf(Taskd.not_available(), "Taskd binary not available") class ServerTestCase(unittest.TestCase): @classmethod def setUpClass(cls): From 04f5f7e2a8aafb22487c5d1943b628fb983fa592 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 01:37:17 +0100 Subject: [PATCH 07/11] Unittest - exit code may be None if process failed to finish --- test/basetest/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/basetest/task.py b/test/basetest/task.py index e7966ff67..72286f22d 100644 --- a/test/basetest/task.py +++ b/test/basetest/task.py @@ -152,7 +152,7 @@ class Task(object): merge_streams=merge_streams, env=self.env) # output[0] is the exit code - if output[0] == 0: + if output[0] == 0 or output[0] is None: raise CommandError(command, *output) return output From 7f9148efb48578f60ac774976c0fb46702879707 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 02:40:56 +0100 Subject: [PATCH 08/11] Unittest - CommandError exception treats SIGABRT specially * SIGABRT will be used to signal processes that failed to finish after the process assigned time (default 1 second). --- test/basetest/exceptions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/basetest/exceptions.py b/test/basetest/exceptions.py index 3bb836711..abfcdb8ae 100644 --- a/test/basetest/exceptions.py +++ b/test/basetest/exceptions.py @@ -1,11 +1,17 @@ # -*- coding: utf-8 -*- +import signal class CommandError(Exception): def __init__(self, cmd, code, out, err, msg=None): if msg is None: - self.msg = ("Command '{0}' finished with unexpected exit code " - "'{1}':\nStdout: '{2}'\nStderr: '{3}'") + if code == signal.SIGABRT: + self.msg = ("Command '{0}' was aborted, likely due to not " + "finishing in due time. The exit code was " + "'{1}':\nStdout: '{2}'\nStderr: '{3}'") + else: + self.msg = ("Command '{0}' finished with unexpected exit code " + "'{1}':\nStdout: '{2}'\nStderr: '{3}'") else: self.msg = msg From e3d0d2ff344f0d617d64b50d9f369c43718d06eb Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 02:43:57 +0100 Subject: [PATCH 09/11] Unittest - Stream blocking tests can now be safely performed * Processes that blocked waiting for stdin data will now be aborted after a 1 second timeout. * As a side-effect any process that takes longer than 1 second to finish will also be aborted. --- test/basetest/utils.py | 79 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/test/basetest/utils.py b/test/basetest/utils.py index 77fd1b21b..15ae52d08 100644 --- a/test/basetest/utils.py +++ b/test/basetest/utils.py @@ -1,11 +1,77 @@ # -*- coding: utf-8 -*- +from __future__ import division import os import sys import socket +import signal from subprocess import Popen, PIPE, STDOUT +from threading import Thread +from Queue import Queue, Empty +from time import sleep from .exceptions import CommandError USED_PORTS = set() +ON_POSIX = 'posix' in sys.builtin_module_names + + +def wait_process(proc, timeout=1): + """Wait for process to finish + """ + sleeptime = .1 + # Max number of attempts until giving up + tries = int(timeout / sleeptime) + + # Wait for up to a second for the process to finish and avoid zombies + for i in range(tries): + exit = proc.poll() + + if exit is not None: + break + + sleep(sleeptime) + + return exit + + +def _get_output(proc, input): + """Collect output from the subprocess without blocking the main process if + subprocess hangs. + """ + def queue_output(proc, input, outq, errq): + """Read/Write output/input of given process. + This function is meant to be executed in a thread as it may block + """ + # Send input and wait for finish + out, err = proc.communicate(input) + # Give the output back to the caller + outq.put(out) + errq.put(err) + + outq = Queue() + errq = Queue() + + t = Thread(target=queue_output, args=(proc, input, outq, errq)) + t.daemon = True + t.start() + + # A task process shouldn't take longer than 1 second to finish + exit = wait_process(proc) + + # If it does take longer than 1 second, abort it + if exit is None: + proc.send_signal(signal.SIGABRT) + exit = wait_process(proc) + + try: + out = outq.get_nowait() + except Empty: + out = None + try: + err = errq.get_nowait() + except Empty: + err = None + + return out, err def run_cmd_wait(cmd, input=None, stdout=PIPE, stderr=PIPE, @@ -22,16 +88,9 @@ def run_cmd_wait(cmd, input=None, stdout=PIPE, stderr=PIPE, else: stderr = PIPE - p = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=env) - out, err = p.communicate(input) - - # In python3 we will be able use the following instead of the previous - # line to avoid locking if task is unexpectedly waiting for input - # try: - # out, err = p.communicate(input, timeout=15) - # except TimeoutExpired: - # p.kill() - # out, err = proc.communicate() + p = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=1, + close_fds=ON_POSIX, env=env) + out, err = _get_output(p, input) if p.returncode != 0: raise CommandError(cmd, p.returncode, out, err) From ea5186716e64e8feec76eb114b12935578b809fb Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 03:02:47 +0100 Subject: [PATCH 10/11] Unittest - Example of TAP diag use in template.t * Also prettify its output with the amazing whitespace --- test/basetest/testing.py | 2 +- test/template.t | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/basetest/testing.py b/test/basetest/testing.py index b8e03e9c7..e90f81b46 100644 --- a/test/basetest/testing.py +++ b/test/basetest/testing.py @@ -8,7 +8,7 @@ class TestCase(unittest.TestCase): def diag(self, out): sys.stdout.write("# --- diag start ---\n") for line in out.split("\n"): - sys.stdout.write("#" + line + "\n") + sys.stdout.write("# " + line + "\n") sys.stdout.write("# --- diag end ---\n") diff --git a/test/template.t b/test/template.t index 6bfb83a43..0851dafa8 100755 --- a/test/template.t +++ b/test/template.t @@ -34,6 +34,9 @@ class TestBugNumber(TestCase): expected = "Copyright \(C\) \d{4} - %d" % (datetime.now().year,) self.assertRegexpMatches(out.decode("utf8"), expected) + # TAP diagnostics on the bas + self.diag("Yay TAP diagnostics") + def test_fail_other(self): """Nothing to do with Copyright""" self.assertEqual("I like to code", "I like\nto code\n") From 9bd7b336f94805982e81192b6a94f840391f1606 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Tue, 15 Jul 2014 03:04:17 +0100 Subject: [PATCH 11/11] Unittest - Don't use unittest.TestCase use basetest.TestCase instead --- test/template.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/template.t b/test/template.t index 0851dafa8..cb081eaf9 100755 --- a/test/template.t +++ b/test/template.t @@ -54,7 +54,7 @@ class TestBugNumber(TestCase): @unittest.skipIf(Taskd.not_available(), "Taskd binary not available") -class ServerTestCase(unittest.TestCase): +class ServerTestCase(TestCase): @classmethod def setUpClass(cls): cls.taskd = Taskd()