mirror of
https://github.com/GothenburgBitFactory/taskshell.git
synced 2025-06-26 10:54:29 +02:00
Test: Upgraded test framework
- Uses Taskwarrior's parallel harness. - Uses Taskwarrior's 'problems' script for diagnosis. - Removed 'run_all.in' processing. - Removed Perl 'template.t' script.
This commit is contained in:
parent
113d96f382
commit
3ceab3f7a4
6 changed files with 358 additions and 138 deletions
154
test/problems
154
test/problems
|
@ -1,39 +1,133 @@
|
|||
#!/usr/bin/env perl
|
||||
#!/usr/bin/env python
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
|
||||
if (open my $fh, '<', 'all.log')
|
||||
{
|
||||
my $test_file;
|
||||
my %errors;
|
||||
my %skipped;
|
||||
my %expected;
|
||||
|
||||
while (my $line = <$fh>)
|
||||
{
|
||||
$test_file = $1 if $line =~ /^# (\S+\.t)$/;
|
||||
$errors{$test_file}++ if $line =~ /^not /;
|
||||
$skipped{$test_file}++ if $line =~ /^skip /;
|
||||
$expected{$test_file}++ if $line =~ /^# EXPECTED_FAILURE: /;
|
||||
}
|
||||
def color(text, c):
|
||||
"""
|
||||
Add color on the keyword that identifies the state of the test
|
||||
"""
|
||||
if sys.stdout.isatty():
|
||||
clear = "\033[0m"
|
||||
|
||||
close $fh;
|
||||
colors = {
|
||||
"red": "\033[1m\033[91m",
|
||||
"yellow": "\033[1m\033[93m",
|
||||
"green": "\033[1m\033[92m",
|
||||
}
|
||||
return colors[c] + text + clear
|
||||
else:
|
||||
return text
|
||||
|
||||
print "Failed\n";
|
||||
printf "%-32s %4d\n", $_, $errors{$_}
|
||||
for sort {$errors{$b} <=> $errors{$a}} keys %errors;
|
||||
|
||||
print "\n";
|
||||
print "Skipped\n";
|
||||
printf "%-32s %4d\n", $_, $skipped{$_}
|
||||
for sort {$skipped{$b} <=> $skipped{$a}} keys %skipped;
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Report on test results")
|
||||
parser.add_argument('--summary', action="store_true",
|
||||
help="Display only the totals in each category")
|
||||
parser.add_argument('tapfile', default="all.log", nargs="?",
|
||||
help="File containing TAP output")
|
||||
return parser.parse_args()
|
||||
|
||||
print "\n";
|
||||
print "Expected failures (part of skipped)\n";
|
||||
printf "%-32s %4d\n", $_, $expected{$_}
|
||||
for sort {$expected{$b} <=> $expected{$a}} keys %expected;
|
||||
}
|
||||
|
||||
exit 0;
|
||||
def print_category(tests):
|
||||
if not cmd_args.summary:
|
||||
for key in sorted(tests):
|
||||
print("%-32s %4d" % (key, tests[key]))
|
||||
|
||||
|
||||
def pad(i):
|
||||
return " " * i
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd_args = parse_args()
|
||||
|
||||
errors = defaultdict(int)
|
||||
skipped = defaultdict(int)
|
||||
expected = defaultdict(int)
|
||||
unexpected = defaultdict(int)
|
||||
passed = defaultdict(int)
|
||||
|
||||
file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$")
|
||||
timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$")
|
||||
start = None
|
||||
stop = None
|
||||
|
||||
with open(cmd_args.tapfile) as fh:
|
||||
for line in fh:
|
||||
if start is None:
|
||||
# First line contains the starting timestamp
|
||||
start = float(timestamp.match(line).group(1))
|
||||
continue
|
||||
|
||||
match = file.match(line)
|
||||
if match:
|
||||
filename = match.group(1)
|
||||
|
||||
if line.startswith("ok "):
|
||||
passed[filename] += 1
|
||||
|
||||
if line.startswith("not "):
|
||||
errors[filename] += 1
|
||||
|
||||
if line.startswith("skip "):
|
||||
skipped[filename] += 1
|
||||
|
||||
if line.startswith("# EXPECTED_FAILURE:"):
|
||||
expected[filename] += 1
|
||||
|
||||
if line.startswith("# UNEXPECTED_SUCCESS:"):
|
||||
unexpected[filename] += 1
|
||||
|
||||
# Last line contains the ending timestamp
|
||||
stop = float(timestamp.match(line).group(1))
|
||||
|
||||
# Remove expected failures from the skipped tests category
|
||||
for filename, value in expected.items():
|
||||
if skipped[filename] == value:
|
||||
del skipped[filename]
|
||||
else:
|
||||
skipped[filename] -= value
|
||||
|
||||
v = "{0:>5d}"
|
||||
passed_str = "Passed:" + pad(24)
|
||||
passed_int = v.format(sum(passed.values()))
|
||||
error_str = "Failed:" + pad(24)
|
||||
error_int = v.format(sum(errors.values()))
|
||||
unexpected_str = "Unexpected successes:" + pad(10)
|
||||
unexpected_int = v.format(sum(unexpected.values()))
|
||||
skipped_str = "Skipped:" + pad(23)
|
||||
skipped_int = v.format(sum(skipped.values()))
|
||||
expected_str = "Expected failures:" + pad(13)
|
||||
expected_int = v.format(sum(expected.values()))
|
||||
runtime_str = "Runtime:" + pad(20)
|
||||
runtime_int = "{0:>8.2f} seconds".format(stop - start)
|
||||
|
||||
if cmd_args.summary:
|
||||
print(color(passed_str, "green"), passed_int)
|
||||
print(color(error_str, "red"), error_int)
|
||||
print(color(unexpected_str, "red"), unexpected_int)
|
||||
print(color(skipped_str, "yellow"), skipped_int)
|
||||
print(color(expected_str, "yellow"), expected_int)
|
||||
print(runtime_str, runtime_int)
|
||||
|
||||
else:
|
||||
print(color(error_str, "red"))
|
||||
print_category(errors)
|
||||
print()
|
||||
print(color(unexpected_str, "red"))
|
||||
print_category(unexpected)
|
||||
print()
|
||||
print(color(skipped_str, "yellow"))
|
||||
print_category(skipped)
|
||||
print()
|
||||
print(color(expected_str, "yellow"))
|
||||
print_category(expected)
|
||||
|
||||
# If we encoutered any failures, return non-zero code
|
||||
if sum(errors.values()):
|
||||
sys.exit(1)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue