From 7ac3ea424bd671a62918fffebffb1dedc5f43336 Mon Sep 17 00:00:00 2001 From: Vince Harron Date: Fri, 26 Jun 2015 15:13:21 +0000 Subject: [PATCH] Added expectedFlakey test decorator SUMMARY Flakey tests get two chances to pass Also, switched a bunch of tests to use new decorator. TEST PLAN Add one of these decorators to a test Edit a test to pass on the first invocation, confirm test appears as pass Edit a test to pass on the first invocation, pass on the second, confirm test appears as xfail Edit a test to fail on two consecutive runs, confirm test appears in results as fail/error Differential Revision: http://reviews.llvm.org/D10721 llvm-svn: 240789 --- .../call-function/TestCallStopAndContinue.py | 4 +- .../call-restarts/TestCallThatRestarts.py | 2 +- .../timeout/TestCallWithTimeout.py | 2 +- .../attach_resume/TestAttachResume.py | 2 +- .../avoids-fd-leak/TestFdLeak.py | 4 +- .../TestExprDoesntBlock.py | 2 +- .../stop-hook/TestStopHookMechanism.py | 2 +- .../TestStopHookMultipleThreads.py | 2 +- .../TestCreateAfterAttach.py | 3 + .../TestThreadSpecificBreakpoint.py | 2 +- .../TestRegisterVariables.py | 2 +- lldb/test/lang/cpp/this/TestCPPThis.py | 2 +- lldb/test/lldbtest.py | 62 +++++++++++++++++++ .../test/python_api/signals/TestSignalsAPI.py | 2 +- lldb/test/types/TestFloatTypes.py | 2 +- 15 files changed, 80 insertions(+), 15 deletions(-) diff --git a/lldb/test/expression_command/call-function/TestCallStopAndContinue.py b/lldb/test/expression_command/call-function/TestCallStopAndContinue.py index aa0e532778e8..a012f994b7a8 100644 --- a/lldb/test/expression_command/call-function/TestCallStopAndContinue.py +++ b/lldb/test/expression_command/call-function/TestCallStopAndContinue.py @@ -22,14 +22,14 @@ class ExprCommandCallStopContinueTestCase(TestBase): @skipUnlessDarwin @dsym_test - @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX + @expectedFlakeyDarwin("llvm.org/pr20274") def test_with_dsym(self): """Test gathering result from interrupted function call.""" self.buildDsym() self.call_function() @dwarf_test - @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX + @expectedFlakeyDarwin("llvm.org/pr20274") def test_with_dwarf(self): """Test gathering result from interrupted function call.""" self.buildDwarf() diff --git a/lldb/test/expression_command/call-restarts/TestCallThatRestarts.py b/lldb/test/expression_command/call-restarts/TestCallThatRestarts.py index f6f24c15430b..ba8890a04ce6 100644 --- a/lldb/test/expression_command/call-restarts/TestCallThatRestarts.py +++ b/lldb/test/expression_command/call-restarts/TestCallThatRestarts.py @@ -29,7 +29,7 @@ class ExprCommandThatRestartsTestCase(TestBase): @dwarf_test @skipIfFreeBSD # llvm.org/pr19246: intermittent failure - @expectedFailureLinux("llvm.org/pr19246") # intermittent failure + @expectedFailureLinux("llvm.org/pr19246") @skipIfDarwin # llvm.org/pr19246: intermittent failure @skipIfWindows # Test relies on signals, unsupported on Windows def test_with_dwarf(self): diff --git a/lldb/test/expression_command/timeout/TestCallWithTimeout.py b/lldb/test/expression_command/timeout/TestCallWithTimeout.py index 4b13316ad0b9..4cc557743647 100644 --- a/lldb/test/expression_command/timeout/TestCallWithTimeout.py +++ b/lldb/test/expression_command/timeout/TestCallWithTimeout.py @@ -28,7 +28,7 @@ class ExprCommandWithTimeoutsTestCase(TestBase): self.call_function() @expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot - @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux + @expectedFlakeyLinux("llvm.org/pr20275") @dwarf_test def test_with_dwarf(self): """Test calling std::String member function.""" diff --git a/lldb/test/functionalities/attach_resume/TestAttachResume.py b/lldb/test/functionalities/attach_resume/TestAttachResume.py index 1bf551c801f7..7331154a4374 100644 --- a/lldb/test/functionalities/attach_resume/TestAttachResume.py +++ b/lldb/test/functionalities/attach_resume/TestAttachResume.py @@ -22,7 +22,7 @@ class AttachResumeTestCase(TestBase): self.buildDwarf() self.process_attach_continue_interrupt_detach() - @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs + @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs @skipIfRemote def process_attach_continue_interrupt_detach(self): """Test attach/continue/interrupt/detach""" diff --git a/lldb/test/functionalities/avoids-fd-leak/TestFdLeak.py b/lldb/test/functionalities/avoids-fd-leak/TestFdLeak.py index c992ffde984a..9f254398a531 100644 --- a/lldb/test/functionalities/avoids-fd-leak/TestFdLeak.py +++ b/lldb/test/functionalities/avoids-fd-leak/TestFdLeak.py @@ -41,8 +41,8 @@ class AvoidsFdLeakTestCase(TestBase): self.assertTrue(process.GetExitStatus() == 0, "Process returned non-zero status. Were incorrect file descriptors passed?") - @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd - @expectedFailureLinux # xfail flakey test to get buildbot green + @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd + @expectedFlakeyLinux @skipIfWindows # The check for descriptor leakage needs to be implemented differently here. @skipIfTargetAndroid() # Android have some other file descriptors open by the shell def test_fd_leak_multitarget (self): diff --git a/lldb/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py b/lldb/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py index dd19a3f59e12..3c8de16fa5a6 100644 --- a/lldb/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py +++ b/lldb/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py @@ -24,7 +24,7 @@ class ExprDoesntDeadlockTestCase(TestBase): @dwarf_test @expectedFailureFreeBSD('llvm.org/pr17946') - @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid() + @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid() def test_with_dwarf_and_run_command(self): """Test that expr will time out and allow other threads to run if it blocks.""" self.buildDwarf() diff --git a/lldb/test/functionalities/stop-hook/TestStopHookMechanism.py b/lldb/test/functionalities/stop-hook/TestStopHookMechanism.py index 87be6d9bba98..0d1125057efb 100644 --- a/lldb/test/functionalities/stop-hook/TestStopHookMechanism.py +++ b/lldb/test/functionalities/stop-hook/TestStopHookMechanism.py @@ -19,7 +19,7 @@ class StopHookMechanismTestCase(TestBase): self.stop_hook_firing() @skipIfFreeBSD # llvm.org/pr15037 - @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux + @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows") @dwarf_test def test_with_dwarf(self): diff --git a/lldb/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py b/lldb/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py index 1792bf994cb6..9e7d7ffae906 100644 --- a/lldb/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py +++ b/lldb/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py @@ -21,7 +21,7 @@ class StopHookForMultipleThreadsTestCase(TestBase): @dwarf_test @expectedFailureFreeBSD("llvm.org/pr15037") - @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux + @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows") @skipIfFreeBSD # llvm.org/pr22784 def test_stop_hook_multiple_threads_with_dwarf(self): diff --git a/lldb/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py b/lldb/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py index 378c6a2c2c04..a3e01312697e 100644 --- a/lldb/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py +++ b/lldb/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py @@ -22,6 +22,8 @@ class CreateAfterAttachTestCase(TestBase): @skipIfFreeBSD # Hangs. May be the same as Linux issue llvm.org/pr16229 but # not yet investigated. Revisit once required functionality # is implemented for FreeBSD. + @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux + @dwarf_test def test_create_after_attach_with_dwarf_and_popen(self): """Test thread creation after process attach.""" @@ -32,6 +34,7 @@ class CreateAfterAttachTestCase(TestBase): # for FreeBSD. @dwarf_test @skipIfRemote + @expectedFlakeyLinux("llvm.org/pr16229") # 1/100 dosep, build 3546, clang-3.5 x84_64 def test_create_after_attach_with_dwarf_and_fork(self): """Test thread creation after process attach.""" self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False)) diff --git a/lldb/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py b/lldb/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py index c3525c37ad69..3faed3fe6b3f 100644 --- a/lldb/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py +++ b/lldb/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py @@ -23,7 +23,7 @@ class ThreadSpecificBreakTestCase(TestBase): @expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun @python_api_test @dwarf_test - @expectedFailureLinux # this test fails 6/100 dosep runs + @expectedFlakeyLinux # this test fails 6/100 dosep runs def test_with_dwarf_python(self): """Test that we obey thread conditioned breakpoints.""" self.buildDwarf() diff --git a/lldb/test/lang/c/register_variables/TestRegisterVariables.py b/lldb/test/lang/c/register_variables/TestRegisterVariables.py index c35497c3b7cb..5c754d289306 100644 --- a/lldb/test/lang/c/register_variables/TestRegisterVariables.py +++ b/lldb/test/lang/c/register_variables/TestRegisterVariables.py @@ -19,7 +19,7 @@ class RegisterVariableTestCase(TestBase): @dwarf_test @expectedFailureClang(None, ['<', '3.5']) - @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2 + @expectedFlakeyGcc # failed with gcc4.8.2 def test_with_dwarf_and_run_command(self): """Test expressions on register values.""" self.buildDwarf() diff --git a/lldb/test/lang/cpp/this/TestCPPThis.py b/lldb/test/lang/cpp/this/TestCPPThis.py index 32d60779f1a3..b6621e56d536 100644 --- a/lldb/test/lang/cpp/this/TestCPPThis.py +++ b/lldb/test/lang/cpp/this/TestCPPThis.py @@ -22,7 +22,7 @@ class CPPThisTestCase(TestBase): @expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function. @expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions @dwarf_test - @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7 + @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7 def test_with_dwarf_and_run_command(self): """Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods""" self.buildDwarf() diff --git a/lldb/test/lldbtest.py b/lldb/test/lldbtest.py index f2333e3a727c..c1383346c592 100644 --- a/lldb/test/lldbtest.py +++ b/lldb/test/lldbtest.py @@ -712,6 +712,68 @@ def expectedFailureAndroid(bugnumber=None, api_levels=None): return expectedFailure(fn, bugnumber) +# if the test passes on the first try, we're done (success) +# if the test fails once, then passes on the second try, raise an ExpectedFailure +# if the test fails twice in a row, re-throw the exception from the second test run +def expectedFlakey(expected_fn, bugnumber=None): + def expectedFailure_impl(func): + @wraps(func) + def wrapper(*args, **kwargs): + from unittest2 import case + self = args[0] + try: + func(*args, **kwargs) + except Exception: + if expected_fn(self): + # retry + try: + func(*args, **kwargs) + except Exception: + # oh snap! two failures in a row, record a failure/error + raise + # record the expected failure + raise case._ExpectedFailure(sys.exc_info(), bugnumber) + else: + raise + return wrapper + # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments + # return decorator in this case, so it will be used to decorating original method + if callable(bugnumber): + return expectedFailure_impl(bugnumber) + else: + return expectedFailure_impl + +def expectedFlakeyOS(oslist, bugnumber=None, compilers=None): + def fn(self): + return (self.getPlatform() in oslist and + self.expectedCompiler(compilers)) + return expectedFlakey(fn, bugnumber) + +def expectedFlakeyDarwin(bugnumber=None, compilers=None): + # For legacy reasons, we support both "darwin" and "macosx" as OS X triples. + return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers) + +def expectedFlakeyLinux(bugnumber=None, compilers=None): + return expectedFlakeyOS(['linux'], bugnumber, compilers) + +def expectedFlakeyFreeBSD(bugnumber=None, compilers=None): + return expectedFlakeyOS(['freebsd'], bugnumber, compilers) + +def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None): + if compiler_version is None: + compiler_version=['=', None] + def fn(self): + return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version) + return expectedFlakey(fn, bugnumber) + +# @expectedFlakeyClang('bugnumber', ['<=', '3.4']) +def expectedFlakeyClang(bugnumber=None, compiler_version=None): + return expectedFlakeyCompiler('clang', compiler_version, bugnumber) + +# @expectedFlakeyGcc('bugnumber', ['<=', '3.4']) +def expectedFlakeyGcc(bugnumber=None, compiler_version=None): + return expectedFlakeyCompiler('gcc', compiler_version, bugnumber) + def skipIfRemote(func): """Decorate the item to skip tests if testing remotely.""" if isinstance(func, type) and issubclass(func, unittest2.TestCase): diff --git a/lldb/test/python_api/signals/TestSignalsAPI.py b/lldb/test/python_api/signals/TestSignalsAPI.py index 804daefafe88..8d05d3771fdd 100644 --- a/lldb/test/python_api/signals/TestSignalsAPI.py +++ b/lldb/test/python_api/signals/TestSignalsAPI.py @@ -12,7 +12,7 @@ class SignalsAPITestCase(TestBase): mydir = TestBase.compute_mydir(__file__) @python_api_test - @expectedFailureLinux # this test fails 1/100 dosep runs + @expectedFlakeyLinux # this test fails 1/100 dosep runs def test_ignore_signal(self): """Test Python SBUnixSignals.Suppress/Stop/Notify() API.""" self.buildDefault() diff --git a/lldb/test/types/TestFloatTypes.py b/lldb/test/types/TestFloatTypes.py index 9f1e41ddfb7a..7f403fecd7ae 100644 --- a/lldb/test/types/TestFloatTypes.py +++ b/lldb/test/types/TestFloatTypes.py @@ -38,7 +38,7 @@ class FloatTypesTestCase(AbstractBase.GenericTester): @skipUnlessDarwin @dsym_test - @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110 + @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110 def test_double_type_with_dsym(self): """Test that double-type variables are displayed correctly.""" self.build_and_run('double.cpp', set(['double']))