Added expectedFlakey test decorator

SUMMARY
Flakey tests get two chances to pass

Also, switched a bunch of tests to use new decorator.

TEST PLAN
Add one of these decorators to a test
Edit a test to pass on the first invocation, confirm test appears as pass
Edit a test to pass on the first invocation, pass on the second, confirm test appears as xfail
Edit a test to fail on two consecutive runs, confirm test appears in results as fail/error

Differential Revision: http://reviews.llvm.org/D10721

llvm-svn: 240789

lldb/test/expression_command/call-function/TestCallStopAndContinue.py

@@ -22,14 +22,14 @@ class ExprCommandCallStopContinueTestCase(TestBase):
 
     @skipUnlessDarwin
     @dsym_test
-    @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+    @expectedFlakeyDarwin("llvm.org/pr20274")
     def test_with_dsym(self):
         """Test gathering result from interrupted function call."""
         self.buildDsym()
         self.call_function()
 
     @dwarf_test
-    @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+    @expectedFlakeyDarwin("llvm.org/pr20274")
     def test_with_dwarf(self):
         """Test gathering result from interrupted function call."""
         self.buildDwarf()

lldb/test/expression_command/call-restarts/TestCallThatRestarts.py

@@ -29,7 +29,7 @@ class ExprCommandThatRestartsTestCase(TestBase):
 
     @dwarf_test
     @skipIfFreeBSD # llvm.org/pr19246: intermittent failure
-    @expectedFailureLinux("llvm.org/pr19246") # intermittent failure
+    @expectedFailureLinux("llvm.org/pr19246")
     @skipIfDarwin # llvm.org/pr19246: intermittent failure
     @skipIfWindows # Test relies on signals, unsupported on Windows
     def test_with_dwarf(self):

lldb/test/expression_command/timeout/TestCallWithTimeout.py

@@ -28,7 +28,7 @@ class ExprCommandWithTimeoutsTestCase(TestBase):
         self.call_function()
 
     @expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot
-    @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux
+    @expectedFlakeyLinux("llvm.org/pr20275")
     @dwarf_test
     def test_with_dwarf(self):
         """Test calling std::String member function."""

lldb/test/functionalities/attach_resume/TestAttachResume.py

@@ -22,7 +22,7 @@ class AttachResumeTestCase(TestBase):
         self.buildDwarf()
         self.process_attach_continue_interrupt_detach()
 
-    @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs
+    @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs
     @skipIfRemote
     def process_attach_continue_interrupt_detach(self):
         """Test attach/continue/interrupt/detach"""

lldb/test/functionalities/avoids-fd-leak/TestFdLeak.py

@@ -41,8 +41,8 @@ class AvoidsFdLeakTestCase(TestBase):
         self.assertTrue(process.GetExitStatus() == 0,
                 "Process returned non-zero status. Were incorrect file descriptors passed?")
 
-    @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
-    @expectedFailureLinux # xfail flakey test to get buildbot green
+    @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
+    @expectedFlakeyLinux
     @skipIfWindows # The check for descriptor leakage needs to be implemented differently here.
     @skipIfTargetAndroid() # Android have some other file descriptors open by the shell
     def test_fd_leak_multitarget (self):

lldb/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py

@@ -24,7 +24,7 @@ class ExprDoesntDeadlockTestCase(TestBase):
 
     @dwarf_test
     @expectedFailureFreeBSD('llvm.org/pr17946')
-    @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid()
+    @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid()
     def test_with_dwarf_and_run_command(self):
         """Test that expr will time out and allow other threads to run if it blocks."""
         self.buildDwarf()

lldb/test/functionalities/stop-hook/TestStopHookMechanism.py

@@ -19,7 +19,7 @@ class StopHookMechanismTestCase(TestBase):
         self.stop_hook_firing()
 
     @skipIfFreeBSD # llvm.org/pr15037
-    @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
+    @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @dwarf_test
     def test_with_dwarf(self):

lldb/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py

@@ -21,7 +21,7 @@ class StopHookForMultipleThreadsTestCase(TestBase):
 
     @dwarf_test
     @expectedFailureFreeBSD("llvm.org/pr15037")
-    @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
+    @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22784
     def test_stop_hook_multiple_threads_with_dwarf(self):

lldb/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py

@@ -22,6 +22,8 @@ class CreateAfterAttachTestCase(TestBase):
     @skipIfFreeBSD # Hangs.  May be the same as Linux issue llvm.org/pr16229 but
                    # not yet investigated.  Revisit once required functionality
                    # is implemented for FreeBSD.
+    @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux
+
     @dwarf_test
     def test_create_after_attach_with_dwarf_and_popen(self):
         """Test thread creation after process attach."""
@@ -32,6 +34,7 @@ class CreateAfterAttachTestCase(TestBase):
                    # for FreeBSD.
     @dwarf_test
     @skipIfRemote
+    @expectedFlakeyLinux("llvm.org/pr16229") # 1/100 dosep, build 3546, clang-3.5 x84_64
     def test_create_after_attach_with_dwarf_and_fork(self):
         """Test thread creation after process attach."""
         self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False))

lldb/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py

@@ -23,7 +23,7 @@ class ThreadSpecificBreakTestCase(TestBase):
     @expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun
     @python_api_test
     @dwarf_test
-    @expectedFailureLinux # this test fails 6/100 dosep runs
+    @expectedFlakeyLinux # this test fails 6/100 dosep runs
     def test_with_dwarf_python(self):
         """Test that we obey thread conditioned breakpoints."""
         self.buildDwarf()

lldb/test/lang/c/register_variables/TestRegisterVariables.py

@@ -19,7 +19,7 @@ class RegisterVariableTestCase(TestBase):
 
     @dwarf_test
     @expectedFailureClang(None, ['<', '3.5'])
-    @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2
+    @expectedFlakeyGcc # failed with gcc4.8.2
     def test_with_dwarf_and_run_command(self):
         """Test expressions on register values."""
         self.buildDwarf()

lldb/test/lang/cpp/this/TestCPPThis.py

@@ -22,7 +22,7 @@ class CPPThisTestCase(TestBase):
     @expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function.
     @expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions
     @dwarf_test
-    @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7
+    @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7
     def test_with_dwarf_and_run_command(self):
         """Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods"""
         self.buildDwarf()

lldb/test/lldbtest.py

@@ -712,6 +712,68 @@ def expectedFailureAndroid(bugnumber=None, api_levels=None):
 
     return expectedFailure(fn, bugnumber)
 
+# if the test passes on the first try, we're done (success)
+# if the test fails once, then passes on the second try, raise an ExpectedFailure
+# if the test fails twice in a row, re-throw the exception from the second test run
+def expectedFlakey(expected_fn, bugnumber=None):
+    def expectedFailure_impl(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            from unittest2 import case
+            self = args[0]
+            try:
+                func(*args, **kwargs)
+            except Exception:
+                if expected_fn(self):
+                    # retry
+                    try:
+                        func(*args, **kwargs)
+                    except Exception:
+                        # oh snap! two failures in a row, record a failure/error
+                        raise
+                    # record the expected failure
+                    raise case._ExpectedFailure(sys.exc_info(), bugnumber)
+                else:
+                    raise
+        return wrapper
+    # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments
+    # return decorator in this case, so it will be used to decorating original method
+    if callable(bugnumber):
+        return expectedFailure_impl(bugnumber)
+    else:
+        return expectedFailure_impl
+
+def expectedFlakeyOS(oslist, bugnumber=None, compilers=None):
+    def fn(self):
+        return (self.getPlatform() in oslist and
+                self.expectedCompiler(compilers))
+    return expectedFlakey(fn, bugnumber)
+
+def expectedFlakeyDarwin(bugnumber=None, compilers=None):
+    # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
+    return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers)
+
+def expectedFlakeyLinux(bugnumber=None, compilers=None):
+    return expectedFlakeyOS(['linux'], bugnumber, compilers)
+
+def expectedFlakeyFreeBSD(bugnumber=None, compilers=None):
+    return expectedFlakeyOS(['freebsd'], bugnumber, compilers)
+
+def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None):
+    if compiler_version is None:
+        compiler_version=['=', None]
+    def fn(self):
+        return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version)
+    return expectedFlakey(fn, bugnumber)
+
+# @expectedFlakeyClang('bugnumber', ['<=', '3.4'])
+def expectedFlakeyClang(bugnumber=None, compiler_version=None):
+    return expectedFlakeyCompiler('clang', compiler_version, bugnumber)
+
+# @expectedFlakeyGcc('bugnumber', ['<=', '3.4'])
+def expectedFlakeyGcc(bugnumber=None, compiler_version=None):
+    return expectedFlakeyCompiler('gcc', compiler_version, bugnumber)
+
 def skipIfRemote(func):
     """Decorate the item to skip tests if testing remotely."""
     if isinstance(func, type) and issubclass(func, unittest2.TestCase):

lldb/test/python_api/signals/TestSignalsAPI.py

@@ -12,7 +12,7 @@ class SignalsAPITestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @python_api_test
-    @expectedFailureLinux # this test fails 1/100 dosep runs
+    @expectedFlakeyLinux # this test fails 1/100 dosep runs
     def test_ignore_signal(self):
         """Test Python SBUnixSignals.Suppress/Stop/Notify() API."""
         self.buildDefault()

lldb/test/types/TestFloatTypes.py

@@ -38,7 +38,7 @@ class FloatTypesTestCase(AbstractBase.GenericTester):
 
     @skipUnlessDarwin
     @dsym_test
-    @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
+    @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
     def test_double_type_with_dsym(self):
         """Test that double-type variables are displayed correctly."""
         self.build_and_run('double.cpp', set(['double']))