From 18df607dbeb41451f5842f46e9b7a67baf759d5e Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Sun, 26 Jul 2020 11:39:22 +0100
Subject: [PATCH] [lit] Don't include tests skipped due to sharding in reports

When running multiple shards, don't include skipped tests in the xunit
output since merging the files will result in duplicates.
In our CHERI Jenkins CI, I configured the libc++ tests to run using sharding
(since we are testing using a single-CPU QEMU). We then merge the generated
XUnit xml files to produce a final result, but if the individual XMLs
report tests excluded due to sharding each test is included N times in the
final result. This also makes it difficult to find the tests that were
skipped due to missing REQUIRES: etc.

Reviewed By: yln

Differential Revision: https://reviews.llvm.org/D84235
---
 llvm/utils/lit/lit/main.py           | 6 +++++-
 llvm/utils/lit/lit/reports.py        | 2 +-
 llvm/utils/lit/tests/xunit-output.py | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index c47bdede3176..d94d7280809d 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -80,9 +80,13 @@ def main(builtin_params={}):
                              'error.\n')
             sys.exit(2)
 
+    # When running multiple shards, don't include skipped tests in the xunit
+    # output since merging the files will result in duplicates.
+    tests_for_report = discovered_tests
     if opts.shard:
         (run, shards) = opts.shard
         selected_tests = filter_by_shard(selected_tests, run, shards, lit_config)
+        tests_for_report = selected_tests
         if not selected_tests:
             sys.stderr.write('warning: shard does not contain any tests.  '
                              'Consider decreasing the number of shards.\n')
@@ -102,7 +106,7 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     for report in opts.reports:
-        report.write_results(discovered_tests, elapsed)
+        report.write_results(tests_for_report, elapsed)
 
     if lit_config.numErrors:
         sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 9631d5fe3b35..3ce961b44029 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -127,7 +127,7 @@ class XunitReport(object):
     def _get_skip_reason(self, test):
         code = test.result.code
         if code == lit.Test.EXCLUDED:
-            return 'Test not selected (--filter, --max-tests, --run-shard)'
+            return 'Test not selected (--filter, --max-tests)'
         if code == lit.Test.SKIPPED:
             return 'User interrupt'
 
diff --git a/llvm/utils/lit/tests/xunit-output.py b/llvm/utils/lit/tests/xunit-output.py
index 81d8525f33b4..92b693256c69 100644
--- a/llvm/utils/lit/tests/xunit-output.py
+++ b/llvm/utils/lit/tests/xunit-output.py
@@ -14,7 +14,7 @@
 # CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
 # CHECK-NEXT: </testcase>
 # CHECK-NEXT: <testcase classname="test-data.test-data" name="excluded.ini" time="{{[0-1]\.[0-9]+}}">
-# CHECK-NEXT:   <skipped message="Test not selected (--filter, --max-tests, --run-shard)"/>
+# CHECK-NEXT:   <skipped message="Test not selected (--filter, --max-tests)"/>
 # CHECK-NEXT: </testcase>
 # CHECK-NEXT: <testcase classname="test-data.test-data" name="missing_feature.ini" time="{{[0-1]\.[0-9]+}}">
 # CHECK-NEXT:   <skipped message="Missing required feature(s): dummy_feature"/>