maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Refactor test runner to print sub-test-case pass/fail rate. 

llvm-svn: 238467
This commit is contained in:
Zachary Turner 2015-05-28 19:56:26 +00:00
parent ddf1cda2c2
commit c7a7c8acca
4 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lldb/test/api/multithreaded/Makefile
View File

@ -6,4 +6,4 @@ CXX_SOURCES := main.cpp
include $(LEVEL)/Makefile.rules
clean::
rm -rf *.o *.d *.dSYM
rm $(wildcard -rf *.o *.d *.dSYM)

68
lldb/test/dosep.py
View File

@ -66,27 +66,47 @@ default_timeout = os.getenv("LLDB_TEST_TIMEOUT") or "10m"
# Status codes for running command with timeout.
eTimedOut, ePassed, eFailed = 124, 0, 1
def parse_test_results(output):
passes = 0
failures = 0
for result in output:
pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE)
fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE)
error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE)
this_fail_count = 0
this_error_count = 0
if pass_count != None:
passes = passes + int(pass_count.group(1))
if fail_count != None:
failures = failures + int(fail_count.group(1))
if error_count != None:
failures = failures + int(error_count.group(1))
pass
return passes, failures
def call_with_timeout(command, timeout):
"""Run command with a timeout if possible."""
"""-s QUIT will create a coredump if they are enabled on your system"""
process = None
if timeout_command and timeout != "0":
command = [timeout_command, '-s', 'QUIT', timeout] + command
# Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE
if os.name != "nt":
if timeout_command and timeout != "0":
return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
stdin=subprocess.PIPE, close_fds=True)
return (ePassed if subprocess.call(command, stdin=subprocess.PIPE, close_fds=True) == 0
else eFailed)
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
else:
if timeout_command and timeout != "0":
return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
stdin=subprocess.PIPE)
return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0
else eFailed)
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()
exit_status = process.returncode
passes, failures = parse_test_results(output)
return exit_status, passes, failures
def process_dir(root, files, test_root, dotest_argv):
"""Examine a directory for tests, and invoke any found within it."""
timed_out = []
failed = []
passed = []
pass_sub_count = 0
fail_sub_count = 0
for name in files:
path = os.path.join(root, name)
@ -107,15 +127,18 @@ def process_dir(root, files, test_root, dotest_argv):
timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout
exit_status = call_with_timeout(command, timeout)
exit_status, pass_count, fail_count = call_with_timeout(command, timeout)
if ePassed == exit_status:
pass_sub_count = pass_sub_count + pass_count
fail_sub_count = fail_sub_count + fail_count
if exit_status == ePassed:
passed.append(name)
else:
if eTimedOut == exit_status:
timed_out.append(name)
failed.append(name)
return (timed_out, failed, passed)
return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
in_q = None
out_q = None
@ -154,14 +177,18 @@ def walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads):
timed_out = []
failed = []
passed = []
fail_sub_count = 0
pass_sub_count = 0
for test_result in test_results:
(dir_timed_out, dir_failed, dir_passed) = test_result
(dir_timed_out, dir_failed, dir_passed, dir_fail_sub_count, dir_pass_sub_count) = test_result
timed_out += dir_timed_out
failed += dir_failed
passed += dir_passed
fail_sub_count = fail_sub_count + dir_fail_sub_count
pass_sub_count = pass_sub_count + dir_pass_sub_count
return (timed_out, failed, passed)
return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
def getExpectedTimeouts(platform_name):
# returns a set of test filenames that might timeout
@ -294,10 +321,11 @@ Run lldb test suite using a separate process for each test file.
num_threads = 1
system_info = " ".join(platform.uname())
(timed_out, failed, passed) = walk_and_invoke(test_directory, test_subdir, dotest_argv,
num_threads)
(timed_out, failed, passed, all_fails, all_passes) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads)
timed_out = set(timed_out)
num_tests = len(failed) + len(passed)
num_test_files = len(failed) + len(passed)
num_tests = all_fails + all_passes
# move core files into session dir
cores = find('core.*', test_subdir)
@ -322,10 +350,10 @@ Run lldb test suite using a separate process for each test file.
test_name = os.path.splitext(xtime)[0]
touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
print "Ran %d tests." % num_tests
print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), 100.0*len(failed)/num_test_files)
print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, 100.0*all_fails/num_tests)
if len(failed) > 0:
failed.sort()
print "Failing Tests (%d)" % len(failed)
for f in failed:
print "%s: LLDB (suite) :: %s (%s)" % (
"TIMEOUT" if f in timed_out else "FAIL", f, system_info

2
lldb/test/unittest2/result.py
View File

@ -40,6 +40,7 @@ class TestResult(unittest.TestResult):
def __init__(self):
self.failfast = False
self.failures = []
self.passes = []
self.errors = []
self.testsRun = 0
self.skipped = []
@ -117,6 +118,7 @@ class TestResult(unittest.TestResult):
def addSuccess(self, test):
"Called when a test has completed successfully"
self.passes.append(test)
pass
def addSkip(self, test, reason):

35
lldb/test/unittest2/runner.py
View File

@ -175,32 +175,29 @@ class TextTestRunner(unittest.TextTestRunner):
(run, run != 1 and "s" or "", timeTaken))
self.stream.writeln()
expectedFails = unexpectedSuccesses = skipped = 0
expectedFails = unexpectedSuccesses = skipped = passed = failed = errored = 0
try:
results = map(len, (result.expectedFailures,
result.unexpectedSuccesses,
result.skipped))
expectedFails, unexpectedSuccesses, skipped = results
result.skipped,
result.passes,
result.failures,
result.errors))
expectedFails, unexpectedSuccesses, skipped, passed, failed, errored = results
except AttributeError:
pass
infos = []
infos.append("%d passes" % passed)
infos.append("%d failures" % failed)
infos.append("%d errors" % errored)
infos.append("%d skipped" % skipped)
infos.append("%d expected failures" % expectedFails)
infos.append("%d unexpected successes" % unexpectedSuccesses)
self.stream.write("RESULT: ")
if not result.wasSuccessful():
self.stream.write("FAILED")
failed, errored = map(len, (result.failures, result.errors))
if failed:
infos.append("failures=%d" % failed)
if errored:
infos.append("errors=%d" % errored)
else:
self.stream.write("OK")
if skipped:
infos.append("skipped=%d" % skipped)
if expectedFails:
infos.append("expected failures=%d" % expectedFails)
if unexpectedSuccesses:
infos.append("unexpected successes=%d" % unexpectedSuccesses)
if infos:
self.stream.writeln(" (%s)" % (", ".join(infos),))
else:
self.stream.write("\n")
self.stream.write("PASSED")
self.stream.writeln(" (%s)" % (", ".join(infos),))
return result