[lit] Recursively expand substitutions

This allows defining substitutions in terms of other substitutions. For
example, a %build substitution could be defined in terms of a %cxx
substitution as '%cxx %s -o %t.exe' and the script would be properly
expanded.

Differential Revision: https://reviews.llvm.org/D76178

llvm/docs/CommandGuide/lit.rst

@@ -410,11 +410,12 @@ be used to define subdirectories of optional tests, or to change other
 configuration parameters --- for example, to change the test format, or the
 suffixes which identify test files.
 
-PRE-DEFINED SUBSTITUTIONS
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+SUBSTITUTIONS
+~~~~~~~~~~~~~
 
-:program:`lit` provides various patterns that can be used with the RUN command.
-These are defined in TestRunner.py. The base set of substitutions are:
+:program:`lit` allows patterns to be substituted inside RUN commands. It also
+provides the following base set of substitutions, which are defined in
+TestRunner.py:
 
  ======================= ==============
   Macro                   Substitution
@@ -453,6 +454,14 @@ Other substitutions are provided that are variations on this base set and
 further substitution patterns can be defined by each test module. See the
 modules :ref:`local-configuration-files`.
 
+By default, substitutions are expanded exactly once, so that if e.g. a
+substitution ``%build`` is defined in top of another substitution ``%cxx``,
+``%build`` will expand to ``%cxx`` textually, not to what ``%cxx`` expands to.
+However, if the ``recursiveExpansionLimit`` property of the ``LitConfig`` is
+set to a non-negative integer, substitutions will be expanded recursively until
+that limit is reached. It is an error if the limit is reached and expanding
+substitutions again would yield a different result.
+
 More detailed information on substitutions can be found in the
 :doc:`../TestingGuide`.
 

llvm/utils/lit/lit/LitConfig.py

@@ -66,6 +66,19 @@ class LitConfig(object):
         self.maxIndividualTestTime = maxIndividualTestTime
         self.parallelism_groups = parallelism_groups
         self.echo_all_commands = echo_all_commands
+        self._recursiveExpansionLimit = None
+
+    @property
+    def recursiveExpansionLimit(self):
+        return self._recursiveExpansionLimit
+
+    @recursiveExpansionLimit.setter
+    def recursiveExpansionLimit(self, value):
+        if value is not None and not isinstance(value, int):
+            self.fatal('recursiveExpansionLimit must be either None or an integer (got <{}>)'.format(value))
+        if isinstance(value, int) and value < 0:
+            self.fatal('recursiveExpansionLimit must be a non-negative integer (got <{}>)'.format(value))
+        self._recursiveExpansionLimit = value
 
     @property
     def maxIndividualTestTime(self):

llvm/utils/lit/lit/TestRunner.py

@@ -1147,10 +1147,18 @@ def _memoize(f):
 def _caching_re_compile(r):
     return re.compile(r)
 
-def applySubstitutions(script, substitutions):
-    """Apply substitutions to the script.  Allow full regular expression syntax.
+def applySubstitutions(script, substitutions, recursion_limit=None):
+    """
+    Apply substitutions to the script.  Allow full regular expression syntax.
     Replace each matching occurrence of regular expression pattern a with
-    substitution b in line ln."""
+    substitution b in line ln.
+
+    If a substitution expands into another substitution, it is expanded
+    recursively until the line has no more expandable substitutions. If
+    the line can still can be substituted after being substituted
+    `recursion_limit` times, it is an error. If the `recursion_limit` is
+    `None` (the default), no recursive substitution is performed at all.
+    """
     def processLine(ln):
         # Apply substitutions
         for a,b in substitutions:
@@ -1167,9 +1175,28 @@ def applySubstitutions(script, substitutions):
 
         # Strip the trailing newline and any extra whitespace.
         return ln.strip()
+
+    def processLineToFixedPoint(ln):
+        assert isinstance(recursion_limit, int) and recursion_limit >= 0
+        origLine = ln
+        steps = 0
+        processed = processLine(ln)
+        while processed != ln and steps < recursion_limit:
+            ln = processed
+            processed = processLine(ln)
+            steps += 1
+
+        if processed != ln:
+            raise ValueError("Recursive substitution of '%s' did not complete "
+                             "in the provided recursion limit (%s)" % \
+                             (origLine, recursion_limit))
+
+        return processed
+
     # Note Python 3 map() gives an iterator rather than a list so explicitly
     # convert to list before returning.
-    return list(map(processLine, script))
+    process = processLine if recursion_limit is None else processLineToFixedPoint
+    return list(map(process, script))
 
 
 class ParserKind(object):
@@ -1506,7 +1533,8 @@ def executeShTest(test, litConfig, useExternalSh,
     substitutions = list(extra_substitutions)
     substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
                                              normalize_slashes=useExternalSh)
-    script = applySubstitutions(script, substitutions)
+    script = applySubstitutions(script, substitutions,
+                                recursion_limit=litConfig.recursiveExpansionLimit)
 
     # Re-run failed tests up to test.allowed_retries times.
     attempts = test.allowed_retries + 1

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg

@@ -0,0 +1,10 @@
+import lit.formats
+config.name = 'does-not-substitute-no-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py

@@ -0,0 +1 @@
+# RUN: echo %rec5

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg

@@ -0,0 +1,12 @@
+import lit.formats
+config.name = 'does-not-substitute-within-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]
+
+lit_config.recursiveExpansionLimit = 2

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py

@@ -0,0 +1 @@
+# RUN: echo %rec5

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg

@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'negative-integer'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = -4

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg

@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'not-an-integer'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = "not-an-integer"

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg

@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'set-to-none'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = None

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg

@@ -0,0 +1,12 @@
+import lit.formats
+config.name = 'substitutes-within-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]
+
+lit_config.recursiveExpansionLimit = 5

llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py

@@ -0,0 +1 @@
+# RUN: echo %rec5

llvm/utils/lit/tests/shtest-recursive-substitution.py

@@ -0,0 +1,23 @@
+# Check that the config.recursiveExpansionLimit is picked up and will cause
+# lit substitutions to be expanded recursively.
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/substitutes-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
+# CHECK-TEST1: PASS: substitutes-within-limit :: test.py
+# CHECK-TEST1: $ "echo" "STOP"
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
+# CHECK-TEST2: UNRESOLVED: does-not-substitute-within-limit :: test.py
+# CHECK-TEST2: ValueError: Recursive substitution of
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-no-limit --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
+# CHECK-TEST3: PASS: does-not-substitute-no-limit :: test.py
+# CHECK-TEST3: $ "echo" "%rec4"
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/not-an-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST4 %s
+# CHECK-TEST4: recursiveExpansionLimit must be either None or an integer
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/negative-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST5 %s
+# CHECK-TEST5: recursiveExpansionLimit must be a non-negative integer
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s
+# CHECK-TEST6: PASS: set-to-none :: test.py

llvm/utils/lit/tests/unit/TestRunner.py

@@ -199,6 +199,74 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
         except BaseException as e:
             self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
 
+class TestApplySubtitutions(unittest.TestCase):
+    def test_simple(self):
+        script = ["echo %bar"]
+        substitutions = [("%bar", "hello")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["echo hello"])
+
+    def test_multiple_substitutions(self):
+        script = ["echo %bar %baz"]
+        substitutions = [("%bar", "hello"),
+                         ("%baz", "world"),
+                         ("%useless", "shouldnt expand")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["echo hello world"])
+
+    def test_multiple_script_lines(self):
+        script = ["%cxx %compile_flags -c -o %t.o",
+                  "%cxx %link_flags %t.o -o %t.exe"]
+        substitutions = [("%cxx", "clang++"),
+                         ("%compile_flags", "-std=c++11 -O3"),
+                         ("%link_flags", "-lc++")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["clang++ -std=c++11 -O3 -c -o %t.o",
+                                  "clang++ -lc++ %t.o -o %t.exe"])
+
+    def test_recursive_substitution_real(self):
+        script = ["%build %s"]
+        substitutions = [("%cxx", "clang++"),
+                         ("%compile_flags", "-std=c++11 -O3"),
+                         ("%link_flags", "-lc++"),
+                         ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=3)
+        self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"])
+
+    def test_recursive_substitution_limit(self):
+        script = ["%rec5"]
+        # Make sure the substitutions are not in an order where the global
+        # substitution would appear to be recursive just because they are
+        # processed in the right order.
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [5, 6, 7]:
+            result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+            self.assertEqual(result, ["STOP"])
+
+    def test_recursive_substitution_limit_exceeded(self):
+        script = ["%rec5"]
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [0, 1, 2, 3, 4]:
+            try:
+                lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+                self.fail("applySubstitutions should have raised an exception")
+            except ValueError:
+                pass
+
+    def test_recursive_substitution_invalid_value(self):
+        script = ["%rec5"]
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [-1, -2, -3, "foo"]:
+            try:
+                lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+                self.fail("applySubstitutions should have raised an exception")
+            except AssertionError:
+                pass
+
+
 if __name__ == '__main__':
     TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
     unittest.main(verbosity=2)