mirror of
https://github.com/GothenburgBitFactory/taskshell.git
synced 2025-06-26 10:54:29 +02:00
Test: Upgraded test framework
- Uses Taskwarrior's parallel harness. - Uses Taskwarrior's 'problems' script for diagnosis. - Removed 'run_all.in' processing. - Removed Perl 'template.t' script.
This commit is contained in:
Paul Beckingham
parent
113d96f382
commit
3ceab3f7a4
6 changed files with 358 additions and 138 deletions
1
test/.gitignore
vendored
1
test/.gitignore
vendored
|
|
@ -1,4 +1,3 @@
|
|||
run_all
|
||||
all.log
|
||||
color.t
|
||||
fs.t
|
||||
|
|
|
|||
|
|
@ -26,8 +26,6 @@ endif (CYGWIN)
|
|||
|
||||
endif (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
|
||||
|
||||
configure_file (run_all.in run_all)
|
||||
|
||||
add_custom_target (test ./run_all --verbose
|
||||
DEPENDS ${test_SRCS} tasksh_executable
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
|
||||
|
|
|
|||
154
test/problems
154
test/problems
|
|
@ -1,39 +1,133 @@
|
|||
#!/usr/bin/env perl
|
||||
#!/usr/bin/env python
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
|
||||
if (open my $fh, '<', 'all.log')
|
||||
{
|
||||
my $test_file;
|
||||
my %errors;
|
||||
my %skipped;
|
||||
my %expected;
|
||||
|
||||
while (my $line = <$fh>)
|
||||
{
|
||||
$test_file = $1 if $line =~ /^# (\S+\.t)$/;
|
||||
$errors{$test_file}++ if $line =~ /^not /;
|
||||
$skipped{$test_file}++ if $line =~ /^skip /;
|
||||
$expected{$test_file}++ if $line =~ /^# EXPECTED_FAILURE: /;
|
||||
}
|
||||
def color(text, c):
|
||||
"""
|
||||
Add color on the keyword that identifies the state of the test
|
||||
"""
|
||||
if sys.stdout.isatty():
|
||||
clear = "\033[0m"
|
||||
|
||||
close $fh;
|
||||
colors = {
|
||||
"red": "\033[1m\033[91m",
|
||||
"yellow": "\033[1m\033[93m",
|
||||
"green": "\033[1m\033[92m",
|
||||
}
|
||||
return colors[c] + text + clear
|
||||
else:
|
||||
return text
|
||||
|
||||
print "Failed\n";
|
||||
printf "%-32s %4d\n", $_, $errors{$_}
|
||||
for sort {$errors{$b} <=> $errors{$a}} keys %errors;
|
||||
|
||||
print "\n";
|
||||
print "Skipped\n";
|
||||
printf "%-32s %4d\n", $_, $skipped{$_}
|
||||
for sort {$skipped{$b} <=> $skipped{$a}} keys %skipped;
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Report on test results")
|
||||
parser.add_argument('--summary', action="store_true",
|
||||
help="Display only the totals in each category")
|
||||
parser.add_argument('tapfile', default="all.log", nargs="?",
|
||||
help="File containing TAP output")
|
||||
return parser.parse_args()
|
||||
|
||||
print "\n";
|
||||
print "Expected failures (part of skipped)\n";
|
||||
printf "%-32s %4d\n", $_, $expected{$_}
|
||||
for sort {$expected{$b} <=> $expected{$a}} keys %expected;
|
||||
}
|
||||
|
||||
exit 0;
|
||||
def print_category(tests):
|
||||
if not cmd_args.summary:
|
||||
for key in sorted(tests):
|
||||
print("%-32s %4d" % (key, tests[key]))
|
||||
|
||||
|
||||
def pad(i):
|
||||
return " " * i
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd_args = parse_args()
|
||||
|
||||
errors = defaultdict(int)
|
||||
skipped = defaultdict(int)
|
||||
expected = defaultdict(int)
|
||||
unexpected = defaultdict(int)
|
||||
passed = defaultdict(int)
|
||||
|
||||
file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$")
|
||||
timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$")
|
||||
start = None
|
||||
stop = None
|
||||
|
||||
with open(cmd_args.tapfile) as fh:
|
||||
for line in fh:
|
||||
if start is None:
|
||||
# First line contains the starting timestamp
|
||||
start = float(timestamp.match(line).group(1))
|
||||
continue
|
||||
|
||||
match = file.match(line)
|
||||
if match:
|
||||
filename = match.group(1)
|
||||
|
||||
if line.startswith("ok "):
|
||||
passed[filename] += 1
|
||||
|
||||
if line.startswith("not "):
|
||||
errors[filename] += 1
|
||||
|
||||
if line.startswith("skip "):
|
||||
skipped[filename] += 1
|
||||
|
||||
if line.startswith("# EXPECTED_FAILURE:"):
|
||||
expected[filename] += 1
|
||||
|
||||
if line.startswith("# UNEXPECTED_SUCCESS:"):
|
||||
unexpected[filename] += 1
|
||||
|
||||
# Last line contains the ending timestamp
|
||||
stop = float(timestamp.match(line).group(1))
|
||||
|
||||
# Remove expected failures from the skipped tests category
|
||||
for filename, value in expected.items():
|
||||
if skipped[filename] == value:
|
||||
del skipped[filename]
|
||||
else:
|
||||
skipped[filename] -= value
|
||||
|
||||
v = "{0:>5d}"
|
||||
passed_str = "Passed:" + pad(24)
|
||||
passed_int = v.format(sum(passed.values()))
|
||||
error_str = "Failed:" + pad(24)
|
||||
error_int = v.format(sum(errors.values()))
|
||||
unexpected_str = "Unexpected successes:" + pad(10)
|
||||
unexpected_int = v.format(sum(unexpected.values()))
|
||||
skipped_str = "Skipped:" + pad(23)
|
||||
skipped_int = v.format(sum(skipped.values()))
|
||||
expected_str = "Expected failures:" + pad(13)
|
||||
expected_int = v.format(sum(expected.values()))
|
||||
runtime_str = "Runtime:" + pad(20)
|
||||
runtime_int = "{0:>8.2f} seconds".format(stop - start)
|
||||
|
||||
if cmd_args.summary:
|
||||
print(color(passed_str, "green"), passed_int)
|
||||
print(color(error_str, "red"), error_int)
|
||||
print(color(unexpected_str, "red"), unexpected_int)
|
||||
print(color(skipped_str, "yellow"), skipped_int)
|
||||
print(color(expected_str, "yellow"), expected_int)
|
||||
print(runtime_str, runtime_int)
|
||||
|
||||
else:
|
||||
print(color(error_str, "red"))
|
||||
print_category(errors)
|
||||
print()
|
||||
print(color(unexpected_str, "red"))
|
||||
print_category(unexpected)
|
||||
print()
|
||||
print(color(skipped_str, "yellow"))
|
||||
print_category(skipped)
|
||||
print()
|
||||
print(color(expected_str, "yellow"))
|
||||
print_category(expected)
|
||||
|
||||
# If we encoutered any failures, return non-zero code
|
||||
if sum(errors.values()):
|
||||
sys.exit(1)
|
||||
|
|
|
|||
234
test/run_all
Executable file
234
test/run_all
Executable file
|
|
@ -0,0 +1,234 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
from multiprocessing import cpu_count
|
||||
from threading import Thread
|
||||
from subprocess import call, Popen, PIPE
|
||||
|
||||
try:
|
||||
# python 2
|
||||
from Queue import Queue, Empty
|
||||
except ImportError:
|
||||
# python 3
|
||||
from queue import Queue, Empty
|
||||
|
||||
# Look for taskd in $PATH instead of task/src/
|
||||
os.environ["TASKD_USE_PATH"] = "1"
|
||||
|
||||
TIMEOUT = .2
|
||||
|
||||
|
||||
def run_test(testqueue, outqueue, threadname):
|
||||
start = time.time()
|
||||
while True:
|
||||
try:
|
||||
test = testqueue.get(block=True, timeout=TIMEOUT)
|
||||
except Empty:
|
||||
break
|
||||
|
||||
log.info("Running test %s", test)
|
||||
|
||||
try:
|
||||
p = Popen(os.path.abspath(test), stdout=PIPE, stderr=PIPE,
|
||||
env=os.environ)
|
||||
out, err = p.communicate()
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
# Premature end
|
||||
break
|
||||
|
||||
if sys.version_info > (3,):
|
||||
out, err = out.decode('utf-8'), err.decode('utf-8')
|
||||
|
||||
output = ("# {0}\n".format(os.path.basename(test)), out, err)
|
||||
log.debug("Collected output %s", output)
|
||||
outqueue.put(output)
|
||||
|
||||
testqueue.task_done()
|
||||
|
||||
log.warning("Finished %s thread after %s seconds",
|
||||
threadname, round(time.time() - start, 3))
|
||||
|
||||
|
||||
class TestRunner(object):
|
||||
def __init__(self):
|
||||
self.threads = []
|
||||
self.tap = open(cmd_args.tapfile, 'w')
|
||||
|
||||
self._parallelq = Queue()
|
||||
self._serialq = Queue()
|
||||
self._outputq = Queue()
|
||||
|
||||
def _find_tests(self):
|
||||
for test in glob.glob("*.t") + glob.glob("*.t.exe"):
|
||||
if os.access(test, os.X_OK):
|
||||
# Executables only
|
||||
if self._is_parallelizable(test):
|
||||
log.debug("Treating as parallel: %s", test)
|
||||
self._parallelq.put(test)
|
||||
else:
|
||||
log.debug("Treating as serial: %s", test)
|
||||
self._serialq.put(test)
|
||||
else:
|
||||
log.debug("Ignored test %s as it is not executable", test)
|
||||
|
||||
log.info("Parallel tests: %s", self._parallelq.qsize())
|
||||
log.info("Serial tests: %s", self._serialq.qsize())
|
||||
|
||||
def _prepare_threads(self):
|
||||
# Serial thread
|
||||
self.threads.append(
|
||||
Thread(target=run_test, args=(self._serialq, self._outputq, "Serial"))
|
||||
)
|
||||
# Parallel threads
|
||||
self.threads.extend([
|
||||
Thread(target=run_test, args=(self._parallelq, self._outputq, "Parallel"))
|
||||
for i in range(cpu_count())
|
||||
])
|
||||
log.info("Spawned %s threads to run tests", len(self.threads))
|
||||
|
||||
def _start_threads(self):
|
||||
for thread in self.threads:
|
||||
# Threads die when main thread dies
|
||||
log.debug("Starting thread %s", thread)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
def _print_timestamp_to_tap(self):
|
||||
now = time.time()
|
||||
timestamp = "# {0} ==> {1}\n".format(
|
||||
now,
|
||||
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now)),
|
||||
)
|
||||
|
||||
log.debug("Adding timestamp %s to TAP file", timestamp)
|
||||
self.tap.write(timestamp)
|
||||
|
||||
def _is_parallelizable(self, test):
|
||||
if cmd_args.serial:
|
||||
return False
|
||||
|
||||
# This is a pretty weird way to do it, and not realiable.
|
||||
# We are dealing with some binary tests though.
|
||||
with open(test, 'rb') as fh:
|
||||
header = fh.read(100).split(b"\n")
|
||||
if len(header) >= 2 and \
|
||||
((b"/usr/bin/env python" in header[0]) or \
|
||||
(header[1][-14:] == b"bash_tap_tw.sh")):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _get_remaining_tests(self):
|
||||
return self._parallelq.qsize() + self._serialq.qsize()
|
||||
|
||||
def is_running(self):
|
||||
for thread in self.threads:
|
||||
if thread.is_alive():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def start(self):
|
||||
self._find_tests()
|
||||
self._prepare_threads()
|
||||
|
||||
self._print_timestamp_to_tap()
|
||||
|
||||
finished = 0
|
||||
total = self._get_remaining_tests()
|
||||
|
||||
self._start_threads()
|
||||
|
||||
while self.is_running() or not self._outputq.empty():
|
||||
try:
|
||||
outputs = self._outputq.get(block=True, timeout=TIMEOUT)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
log.debug("Outputting to TAP: %s", outputs)
|
||||
|
||||
for output in outputs:
|
||||
self.tap.write(output)
|
||||
|
||||
if cmd_args.verbose:
|
||||
sys.stdout.write(output)
|
||||
|
||||
self._outputq.task_done()
|
||||
finished += 1
|
||||
|
||||
log.warning("Finished %s out of %s tests", finished, total)
|
||||
|
||||
self._print_timestamp_to_tap()
|
||||
|
||||
if not self._parallelq.empty() or not self._serialq.empty():
|
||||
raise RuntimeError(
|
||||
"Something went wrong, not all tests were ran. {0} "
|
||||
"remaining.".format(self._get_remaining_tests()))
|
||||
|
||||
def show_report(self):
|
||||
self.tap.flush()
|
||||
|
||||
log.debug("Calling 'problems --summary' for report")
|
||||
return call([os.path.abspath("problems"), "--summary", cmd_args.tapfile])
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Run Taskwarrior tests")
|
||||
parser.add_argument('--verbose', '-v', action="store_true",
|
||||
help="Also send TAP output to stdout")
|
||||
parser.add_argument('--logging', '-l', action="count",
|
||||
default=0,
|
||||
help="Logging level. -lll is the highest level")
|
||||
parser.add_argument('--serial', action="store_true",
|
||||
help="Do not run tests in parallel")
|
||||
parser.add_argument('--tapfile', default="all.log",
|
||||
help="File to use for TAP output")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
runner = TestRunner()
|
||||
runner.start()
|
||||
|
||||
# If we're producing summary report, propagate the return code
|
||||
if not cmd_args.verbose:
|
||||
return runner.show_report()
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd_args = parse_args()
|
||||
|
||||
if cmd_args.logging == 1:
|
||||
level = logging.WARN
|
||||
elif cmd_args.logging == 2:
|
||||
level = logging.INFO
|
||||
elif cmd_args.logging >= 3:
|
||||
level = logging.DEBUG
|
||||
else:
|
||||
level = logging.ERROR
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
level=level,
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
log.debug("Parsed commandline arguments: %s", cmd_args)
|
||||
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
sys.exit(1)
|
||||
|
||||
# vim: ai sts=4 et sw=4
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
#! /bin/sh
|
||||
|
||||
rc=0
|
||||
if [ x"$1" = x"--verbose" ];
|
||||
then
|
||||
for i in ${TESTBLOB}
|
||||
do
|
||||
echo '#' $i
|
||||
$i > test.log 2>&1
|
||||
while read LINE
|
||||
do
|
||||
echo "$LINE"
|
||||
done < test.log
|
||||
if [ $? -ne 0 ]; then
|
||||
rc=1
|
||||
fi
|
||||
rm test.log
|
||||
done
|
||||
exit $rc
|
||||
else
|
||||
date > all.log
|
||||
|
||||
# Perl is used here to get the time in seconds
|
||||
# because 'date +%s' isn't supported on Solaris.
|
||||
STARTEPOCH=`perl -e 'print time'`
|
||||
|
||||
VRAMSTEG=`which vramsteg`
|
||||
BAR=0
|
||||
if [ -x "$VRAMSTEG" ]; then
|
||||
BAR=1
|
||||
COUNT=0
|
||||
TOTAL=`ls ${TESTBLOB} | wc -l`
|
||||
START=`$VRAMSTEG --now`
|
||||
fi
|
||||
|
||||
for i in ${TESTBLOB}
|
||||
do
|
||||
echo '#' $i >>all.log
|
||||
|
||||
if [ $BAR -eq 1 ]; then
|
||||
$VRAMSTEG --label 'All tests' --min 0 --max $TOTAL --current $COUNT --percentage --start $START --estimate
|
||||
COUNT=`expr $COUNT + 1`
|
||||
fi
|
||||
|
||||
$i >> all.log 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $BAR -eq 1 ]; then
|
||||
$VRAMSTEG --remove
|
||||
fi
|
||||
|
||||
date >> all.log
|
||||
|
||||
ENDEPOCH=`perl -e 'print time'`
|
||||
RUNTIME=`expr $ENDEPOCH - $STARTEPOCH`
|
||||
|
||||
printf "Pass: %5d\n" `grep -c '^ok' all.log`
|
||||
printf "Fail: %5d\n" `grep -c '^not' all.log`
|
||||
printf "Skipped: %5d\n" `grep -c '^skip' all.log`
|
||||
printf "Runtime: %5d seconds\n" $RUNTIME
|
||||
exit $rc
|
||||
fi
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#! /usr/bin/env perl
|
||||
################################################################################
|
||||
##
|
||||
## Copyright 2006 - 2015, Paul Beckingham, Federico Hernandez.
|
||||
##
|
||||
## Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
## of this software and associated documentation files (the "Software"), to deal
|
||||
## in the Software without restriction, including without limitation the rights
|
||||
## to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
## copies of the Software, and to permit persons to whom the Software is
|
||||
## furnished to do so, subject to the following conditions:
|
||||
##
|
||||
## The above copyright notice and this permission notice shall be included
|
||||
## in all copies or substantial portions of the Software.
|
||||
##
|
||||
## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
## OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
## THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
## SOFTWARE.
|
||||
##
|
||||
## http://www.opensource.org/licenses/mit-license.php
|
||||
##
|
||||
################################################################################
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 2;
|
||||
|
||||
use File::Basename;
|
||||
my $ut = basename ($0);
|
||||
|
||||
# Bug <id> - <description>
|
||||
my $output = qx{../src/tasksh --version 2>&1};
|
||||
ok ($? == 0, "$ut: version check");
|
||||
like ($output, qr/^\d\.\d\.\d(?:\.beta\d)?$/ms, "$ut: tasksh version found");
|
||||
|
||||
exit 0;
|
||||
Loading…
Add table
Add a link
Reference in a new issue