[lit] Add basic flaky test retry functionality

The plan is to use this for the sanitizer test suite on Windows.  See
PR24554 for more details on why we need this.

Tested manually by injecting rand() into a sanitizer test and watching
what it does.

llvm-svn: 246704

llvm/utils/lit/lit/Test.py

@@ -27,6 +27,7 @@ class ResultCode(object):
                          (self.name, self.isFailure))
 
 PASS        = ResultCode('PASS', False)
+FLAKYPASS   = ResultCode('FLAKYPASS', False)
 XFAIL       = ResultCode('XFAIL', False)
 FAIL        = ResultCode('FAIL', True)
 XPASS       = ResultCode('XPASS', True)
@@ -253,4 +254,4 @@ class Test:
             xml += "\n\t</failure>\n</testcase>"
         else:
             xml += "/>"
-        return xml
+        return xml

llvm/utils/lit/lit/TestRunner.py

@@ -602,5 +602,17 @@ def executeShTest(test, litConfig, useExternalSh,
         return lit.Test.Result(Test.PASS)
 
     script, tmpBase, execdir = res
-    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
 
+    # Re-run failed tests up to test_retry_attempts times.
+    attempts = 1
+    if hasattr(test.config, 'test_retry_attempts'):
+        attempts += test.config.test_retry_attempts
+    for i in range(attempts):
+        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
+        if res.code != Test.FAIL:
+            break
+    # If we had to run the test more than once, count it as a flaky pass. These
+    # will be printed separately in the test summary.
+    if i > 0 and res.code == Test.PASS:
+        res.code = Test.FLAKYPASS
+    return res

llvm/utils/lit/lit/main.py

@@ -414,6 +414,7 @@ def main(builtinParameters = {}):
         lit.util.printHistogram(test_times, title='Tests')
 
     for name,code in (('Expected Passes    ', lit.Test.PASS),
+                      ('Passes With Retry  ', lit.Test.FLAKYPASS),
                       ('Expected Failures  ', lit.Test.XFAIL),
                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),