[lit] Add --xfail and --filter-out (inverse of --filter)

In semi-automated environments,  XFAILing or filtering out known regressions without actually committing changes or temporarily modifying the test suite can be quite useful.

Reviewed By: yln

Differential Revision: https://reviews.llvm.org/D96662

llvm/docs/CommandGuide/lit.rst

@@ -214,6 +214,21 @@ SELECTION OPTIONS
   of this option, which is especially useful in environments where the call
   to ``lit`` is issued indirectly.
 
+.. option:: --filter-out=REGEXP
+
+  Filter out those tests whose name matches the regular expression specified in
+  ``REGEXP``. The environment variable ``LIT_FILTER_OUT`` can be also used in
+  place of this option, which is especially useful in environments where the
+  call to ``lit`` is issued indirectly.
+
+.. option:: --xfail=LIST
+
+  Treat those tests whose name is in the semicolon separated list ``LIST`` as
+  ``XFAIL``. This can be helpful when one does not want to modify the test
+  suite. The environment variable ``LIT_XFAIL`` can be also used in place of
+  this option, which is especially useful in environments where the call to
+  ``lit`` is issued indirectly.
+
 ADDITIONAL OPTIONS
 ------------------
 

llvm/utils/lit/lit/cl_arguments.py

@@ -158,6 +158,16 @@ def parse_args():
             type=_case_insensitive_regex,
             help="Only run tests with paths matching the given regular expression",
             default=os.environ.get("LIT_FILTER", ".*"))
+    selection_group.add_argument("--filter-out",
+            metavar="REGEX",
+            type=_case_insensitive_regex,
+            help="Filter out tests with paths matching the given regular expression",
+            default=os.environ.get("LIT_FILTER_OUT", "^$"))
+    selection_group.add_argument("--xfail",
+            metavar="LIST",
+            type=_semicolon_list,
+            help="XFAIL tests with paths in the semicolon separated list",
+            default=os.environ.get("LIT_XFAIL", ""))
     selection_group.add_argument("--num-shards",
             dest="numShards",
             metavar="M",
@@ -242,6 +252,10 @@ def _case_insensitive_regex(arg):
         raise _error("invalid regular expression: '{}', {}", arg, reason)
 
 
+def _semicolon_list(arg):
+    return arg.split(';')
+
+
 def _error(desc, *args):
     msg = desc.format(*args)
     return argparse.ArgumentTypeError(msg)

llvm/utils/lit/lit/main.py

@@ -68,7 +68,9 @@ def main(builtin_params={}):
     determine_order(discovered_tests, opts.order)
 
     selected_tests = [t for t in discovered_tests if
-                      opts.filter.search(t.getFullName())]
+        opts.filter.search(t.getFullName()) and not
+        opts.filter_out.search(t.getFullName())]
+
     if not selected_tests:
         sys.stderr.write('error: filter did not match any tests '
                          '(of %d discovered).  ' % len(discovered_tests))
@@ -95,6 +97,8 @@ def main(builtin_params={}):
 
     selected_tests = selected_tests[:opts.max_tests]
 
+    mark_xfail(discovered_tests, opts)
+
     mark_excluded(discovered_tests, selected_tests)
 
     start = time.time()
@@ -189,6 +193,11 @@ def filter_by_shard(tests, run, shards, lit_config):
     return selected_tests
 
 
+def mark_xfail(selected_tests, opts):
+    for t in selected_tests:
+        if os.sep.join(t.path_in_suite) in opts.xfail:
+            t.xfails += '*'
+
 def mark_excluded(discovered_tests, selected_tests):
     excluded_tests = set(discovered_tests) - set(selected_tests)
     result = lit.Test.Result(lit.Test.EXCLUDED)

llvm/utils/lit/tests/Inputs/xfail-cl/false.txt

@@ -0,0 +1 @@
+# RUN: false

llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt

@@ -0,0 +1 @@
+# RUN: false

llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg

@@ -0,0 +1,4 @@
+import lit.formats
+config.name = 'top-level-suite'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()

llvm/utils/lit/tests/Inputs/xfail-cl/true.txt

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/selecting.py

@@ -9,9 +9,12 @@
 # CHECK-BAD-PATH: error: did not discover any tests for provided path(s)
 
 # Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs.
+# Check that we exit with an error if we skip all tests, but allow it with --allow-empty-runs.
 #
 # RUN: not %{lit} --filter 'nonexistent'                    %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s
 # RUN:     %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s
+# RUN: not %{lit} --filter-out '.*'                    %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s
+# RUN:     %{lit} --filter-out '.*' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s
 # CHECK-BAD-FILTER: error: filter did not match any tests (of 5 discovered).
 # CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error.
 # CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified.
@@ -21,6 +24,9 @@
 # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
 # RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
 # RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: %{lit} --filter-out 'test-t[a-z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: %{lit} --filter-out 'test-t[A-Z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: env LIT_FILTER_OUT='test-t[a-z]' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
 # CHECK-FILTER: Testing: 2 of 5 tests
 # CHECK-FILTER: Excluded: 3
 

llvm/utils/lit/tests/xfail-cl.py

@@ -0,0 +1,9 @@
+# Check that regex-XFAILing works and can be configured via env var.
+#
+# RUN: %{lit} --xfail 'false.txt;false2.txt' %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: env LIT_XFAIL='false.txt;false2.txt' %{lit} %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s
+# END:
+# CHECK-FILTER: Testing: 3 tests, 3 workers
+# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false.txt
+# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false2.txt
+# CHECK-FILTER-DAG: PASS: top-level-suite :: true.txt