forked from OSchip/llvm-project
lit: Add --repeat=N option, for running each test N times.
- Currently just useful for timing, although it could be extended as one (bad) way to deal with flaky tests. llvm-svn: 88827
This commit is contained in:
parent
6499e9c625
commit
ad6c15465c
|
@ -149,6 +149,11 @@ List the discovered test suites as part of the standard output.
|
|||
|
||||
Run Tcl scripts internally (instead of converting to shell scripts).
|
||||
|
||||
=item B<--repeat>=I<N>
|
||||
|
||||
Run each test I<N> times. Currently this is primarily useful for timing tests,
|
||||
other results are not collated in any reasonable fashion.
|
||||
|
||||
=back
|
||||
|
||||
=head1 EXIT STATUS
|
||||
|
|
|
@ -54,6 +54,14 @@ class Test:
|
|||
self.output = None
|
||||
# The wall time to execute this test, if timing and once complete.
|
||||
self.elapsed = None
|
||||
# The repeat index of this test, or None.
|
||||
self.index = None
|
||||
|
||||
def copyWithIndex(self, index):
|
||||
import copy
|
||||
res = copy.copy(self)
|
||||
res.index = index
|
||||
return res
|
||||
|
||||
def setResult(self, result, output, elapsed):
|
||||
assert self.result is None, "Test result already set!"
|
||||
|
|
|
@ -367,6 +367,8 @@ def parseIntegratedTestScript(test):
|
|||
execpath = test.getExecPath()
|
||||
execdir,execbase = os.path.split(execpath)
|
||||
tmpBase = os.path.join(execdir, 'Output', execbase)
|
||||
if test.index is not None:
|
||||
tmpBase += '_%d' % test.index
|
||||
|
||||
# We use #_MARKER_# to hide %% while we do the other substitutions.
|
||||
substitutions = [('%%', '#_MARKER_#')]
|
||||
|
|
|
@ -388,6 +388,9 @@ def main():
|
|||
group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
|
||||
help="Don't run Tcl scripts using 'sh'",
|
||||
action="store_false", default=True)
|
||||
group.add_option("", "--repeat", dest="repeatTests", metavar="N",
|
||||
help="Repeat tests N times (for timing)",
|
||||
action="store", default=None, type=int)
|
||||
parser.add_option_group(group)
|
||||
|
||||
(opts, args) = parser.parse_args()
|
||||
|
@ -472,6 +475,11 @@ def main():
|
|||
header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
|
||||
opts.numThreads)
|
||||
|
||||
if opts.repeatTests:
|
||||
tests = [t.copyWithIndex(i)
|
||||
for t in tests
|
||||
for i in range(opts.repeatTests)]
|
||||
|
||||
progressBar = None
|
||||
if not opts.quiet:
|
||||
if opts.succinct and opts.useProgressBar:
|
||||
|
@ -524,11 +532,16 @@ def main():
|
|||
print
|
||||
|
||||
if opts.timeTests:
|
||||
byTime = list(tests)
|
||||
byTime.sort(key = lambda t: t.elapsed)
|
||||
# Collate, in case we repeated tests.
|
||||
times = {}
|
||||
for t in tests:
|
||||
key = t.getFullName()
|
||||
times[key] = times.get(key, 0.) + t.elapsed
|
||||
|
||||
byTime = list(times.items())
|
||||
byTime.sort(key = lambda (name,elapsed): elapsed)
|
||||
if byTime:
|
||||
Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
|
||||
title='Tests')
|
||||
Util.printHistogram(byTime, title='Tests')
|
||||
|
||||
for name,code in (('Expected Passes ', Test.PASS),
|
||||
('Expected Failures ', Test.XFAIL),
|
||||
|
|
Loading…
Reference in New Issue