Allow for tests to be disabled at runtime

Summary:
The current implementation of the test suite allows the user to run
a certain subset of tests using '-p', but does not allow the inverse,
where a user wants to run all but some number of known failing tests.
Implement this functionality.

Reviewers: labath, zturner, tfiala

Subscribers: jingham, sas, lldb-commits

Differential Revision: https://reviews.llvm.org/D24629

llvm-svn: 282298

lldb/packages/Python/lldbsuite/test/configuration.py

@@ -101,6 +101,12 @@ parsable = False
 # our test cases.
 regexp = None
 
+# Sets of tests which are excluded at runtime
+skip_files = None
+skip_methods = None
+xfail_files = None
+xfail_methods = None
+
 # By default, recorded session info for errored/failed test are dumped into its
 # own file under a session directory named after the timestamp of the test suite
 # run.  Use '-s session-dir-name' to specify a specific dir name.

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -26,6 +26,7 @@ import atexit
 import os
 import errno
 import platform
+import re
 import signal
 import socket
 import subprocess
@@ -202,6 +203,48 @@ o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the
     sys.exit(0)
 
 
+def parseExclusion(exclusion_file):
+    """Parse an exclusion file, of the following format, where
+       'skip files', 'skip methods', 'xfail files', and 'xfail methods'
+       are the possible list heading values:
+
+       skip files
+       <file name>
+       <file name>
+
+       xfail methods
+       <method name>
+    """
+    excl_type = None
+    case_type = None
+
+    with open(exclusion_file) as f:
+        for line in f:
+            if not excl_type:
+                [excl_type, case_type] = line.split()
+                continue
+
+            line = line.strip()
+            if not line:
+                excl_type = None
+            elif excl_type == 'skip' and case_type == 'files':
+                if not configuration.skip_files:
+                    configuration.skip_files = []
+                configuration.skip_files.append(line)
+            elif excl_type == 'skip' and case_type == 'methods':
+                if not configuration.skip_methods:
+                    configuration.skip_methods = []
+                configuration.skip_methods.append(line)
+            elif excl_type == 'xfail' and case_type == 'files':
+                if not configuration.xfail_files:
+                    configuration.xfail_files = []
+                configuration.xfail_files.append(line)
+            elif excl_type == 'xfail' and case_type == 'methods':
+                if not configuration.xfail_methods:
+                    configuration.xfail_methods = []
+                configuration.xfail_methods.append(line)
+
+
 def parseOptionsAndInitTestdirs():
     """Initialize the list of directories containing our unittest scripts.
 
@@ -331,6 +374,9 @@ def parseOptionsAndInitTestdirs():
     if args.executable:
         lldbtest_config.lldbExec = os.path.realpath(args.executable)
 
+    if args.excluded:
+        parseExclusion(args.excluded)
+
     if args.p:
         if args.p.startswith('-'):
             usage(parser)
@@ -749,11 +795,15 @@ def setupSysPath():
 def visit_file(dir, name):
     # Try to match the regexp pattern, if specified.
     if configuration.regexp:
-        import re
         if not re.search(configuration.regexp, name):
             # We didn't match the regex, we're done.
             return
 
+    if configuration.skip_files:
+        for file_regexp in configuration.skip_files:
+            if re.search(file_regexp, name):
+                return
+
     # We found a match for our test.  Add it to the suite.
 
     # Update the sys.path first.

lldb/packages/Python/lldbsuite/test/dotest_args.py

@@ -96,6 +96,9 @@ def create_parser():
         '-p',
         metavar='pattern',
         help='Specify a regexp filename pattern for inclusion in the test suite')
+    group.add_argument('--excluded', metavar='exclusion-file', help=textwrap.dedent(
+        '''Specify a file for tests to exclude. File should contain lists of regular expressions for test files or methods,
+                                with each list under a matching header (xfail files, xfail methods, skip files, skip methods)'''))
     group.add_argument(
         '-G',
         '--category',

lldb/packages/Python/lldbsuite/test/test_result.py

@@ -18,6 +18,8 @@ import os
 # Third-party modules
 import unittest2
 
+from unittest2.util import strclass
+
 # LLDB Modules
 from . import configuration
 from lldbsuite.test_event.event_builder import EventBuilder
@@ -124,10 +126,23 @@ class LLDBTestResult(unittest2.TextTestResult):
             test,
             test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
 
+    def checkExclusion(self, exclusion_list, name):
+        if exclusion_list:
+            import re
+            for item in exclusion_list:
+                if re.search(item, name):
+                    return True
+        return False
+
     def startTest(self, test):
         if configuration.shouldSkipBecauseOfCategories(
                 self.getCategoriesForTest(test)):
             self.hardMarkAsSkipped(test)
+        if self.checkExclusion(
+                configuration.skip_methods,
+                test._testMethodName):
+            self.hardMarkAsSkipped(test)
+
         configuration.setCrashInfoHook(
             "%s at %s" %
             (str(test), inspect.getfile(
@@ -145,6 +160,15 @@ class LLDBTestResult(unittest2.TextTestResult):
                 EventBuilder.event_for_start(test))
 
     def addSuccess(self, test):
+        if self.checkExclusion(
+                configuration.xfail_files,
+                strclass(
+                    test.__class__)) or self.checkExclusion(
+                configuration.xfail_methods,
+                test._testMethodName):
+            self.addUnexpectedSuccess(test, None)
+            return
+
         super(LLDBTestResult, self).addSuccess(test)
         if configuration.parsable:
             self.stream.write(
@@ -214,6 +238,15 @@ class LLDBTestResult(unittest2.TextTestResult):
                     test, err))
 
     def addFailure(self, test, err):
+        if self.checkExclusion(
+                configuration.xfail_files,
+                strclass(
+                    test.__class__)) or self.checkExclusion(
+                configuration.xfail_methods,
+                test._testMethodName):
+            self.addExpectedFailure(test, err, None)
+            return
+
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addFailure(test, err)
         method = getattr(test, "markFailure", None)