From 1e2d50936a3bff2c8065a99d06f37b51185fbd81 Mon Sep 17 00:00:00 2001 From: Thomas Preud'homme Date: Fri, 15 Jan 2021 12:47:32 +0000 Subject: [PATCH] Add lit config for dir with standalone tests Some test systems do not use lit for test discovery but only for its substitution and test selection because they use another way of managing test collections, e.g. CTest. This forces those tests to be invoked with lit --no-indirectly-run-check. When a mix of lit version is in use, it requires to detect the availability of that option. This commit provides a new config option standalone_tests to signal a directory made of tests meant to run as standalone. When this option is set, lit skips test discovery and the indirectly run check. It also adds the missing documentation for --no-indirectly-run-check. Reviewed By: jdenny Differential Revision: https://reviews.llvm.org/D94766 --- llvm/docs/CommandGuide/lit.rst | 10 +++++++ llvm/utils/lit/lit/TestingConfig.py | 7 +++-- llvm/utils/lit/lit/discovery.py | 19 ++++++++++-- .../standalone-tests-with-excludes/lit.cfg | 5 ++++ .../standalone-tests-with-excludes/true.txt | 1 + .../standalone-tests-with-suffixes/lit.cfg | 5 ++++ .../standalone-tests-with-suffixes/true.txt | 1 + .../lit/tests/Inputs/standalone-tests/lit.cfg | 4 +++ .../tests/Inputs/standalone-tests/true.txt | 1 + llvm/utils/lit/tests/discovery.py | 30 +++++++++++++++++++ 10 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg create mode 100644 llvm/utils/lit/tests/Inputs/standalone-tests/true.txt diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst index f4d9d9d7e6a1..3820cc446c3c 100644 --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -154,6 +154,11 @@ EXECUTION OPTIONS suite take the most time to execute. Note that this option is most useful with ``-j 1``. +.. option:: --no-indirectly-run-check + + Do not error if a test would not be run if the user had specified the + containing directory instead of naming the test directly. + .. _selection-options: SELECTION OPTIONS @@ -372,6 +377,11 @@ executed, two important global variables are predefined: **environment** A dictionary representing the environment to use when executing tests in the suite. + **standalone_tests** When true, mark a directory with tests expected to be run + standalone. Test discovery is disabled for that directory and + *--no-indirectly-run-check* is in effect. *lit.suffixes* and *lit.excludes* + must be empty when this variable is true. + **suffixes** For **lit** test formats which scan directories for tests, this variable is a list of suffixes to identify test files. Used by: *ShTest*. diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py index 38d05066a2b0..d40ee37636f3 100644 --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -60,7 +60,8 @@ class TestingConfig(object): test_source_root = None, excludes = [], available_features = available_features, - pipefail = True) + pipefail = True, + standalone_tests = False) def load_from_path(self, path, litConfig): """ @@ -105,7 +106,8 @@ class TestingConfig(object): environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False, parallelism_group = None): + is_early = False, parallelism_group = None, + standalone_tests = False): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -118,6 +120,7 @@ class TestingConfig(object): self.excludes = set(excludes) self.available_features = set(available_features) self.pipefail = pipefail + self.standalone_tests = standalone_tests # This list is used by TestRunner.py to restrict running only tests that # require one of the features in this list if this list is non-empty. # Configurations can set this list to restrict the set of tests to run. diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py index 2f027a5b03fe..a185ae676d14 100644 --- a/llvm/utils/lit/lit/discovery.py +++ b/llvm/utils/lit/lit/discovery.py @@ -160,8 +160,13 @@ def getTestsInSuite(ts, path_in_suite, litConfig, # tests which are not executed. The check adds some performance # overhead which might be important if a large number of tests # are being run directly. - # --no-indirectly-run-check: skips this check. - if indirectlyRunCheck and lc.test_format is not None: + # This check can be disabled by using --no-indirectly-run-check or + # setting the standalone_tests variable in the suite's configuration. + if ( + indirectlyRunCheck + and lc.test_format is not None + and not lc.standalone_tests + ): found = False for res in lc.test_format.getTestsInDirectory(ts, test_dir_in_suite, litConfig, lc): @@ -171,6 +176,7 @@ def getTestsInSuite(ts, path_in_suite, litConfig, if not found: litConfig.error( '%r would not be run indirectly: change name or LIT config' + '(e.g. suffixes or standalone_tests variables)' % test.getFullName()) yield test @@ -180,6 +186,15 @@ def getTestsInSuite(ts, path_in_suite, litConfig, # local configuration. lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache) + # Directory contains tests to be run standalone. Do not try to discover. + if lc.standalone_tests: + if lc.suffixes or lc.excludes: + litConfig.warning( + 'standalone_tests set in LIT config but suffixes or excludes' + ' are also set' + ) + return + # Search for tests. if lc.test_format is not None: for res in lc.test_format.getTestsInDirectory(ts, path_in_suite, diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg new file mode 100644 index 000000000000..37d96ddaebfe --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg @@ -0,0 +1,5 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.excludes = ['.test'] +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt new file mode 100644 index 000000000000..b80b60b7a279 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg new file mode 100644 index 000000000000..41434c9855b3 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg @@ -0,0 +1,5 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.suffixes = ['.txt'] +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt new file mode 100644 index 000000000000..b80b60b7a279 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg new file mode 100644 index 000000000000..12ed7b0ce067 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg @@ -0,0 +1,4 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt new file mode 100644 index 000000000000..b80b60b7a279 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py index cb04edaab04c..39743bfd7c58 100644 --- a/llvm/utils/lit/tests/discovery.py +++ b/llvm/utils/lit/tests/discovery.py @@ -148,6 +148,36 @@ # RUN: %{lit} \ # RUN: %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check +# Check that a standalone test with no suffixes set is run without any errors. +# +# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out +# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s +# +# CHECK-STANDALONE: PASS: Standalone tests :: true.txt + +# Check that an error is produced if suffixes variable is set for a suite with +# standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err +# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes + +# Check that an error is produced if excludes variable is set for a suite with +# standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err +# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes + +# Check that no discovery is done for testsuite with standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err +# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s) + # Check that we don't recurse infinitely when loading an site specific test # suite located inside the test source root. #