[lit] Fix internal env calling env

Without this patch, when using lit's internal shell, if `env` on a lit
RUN line calls `env`, lit accidentally searches for the latter as an
external executable.  What's worse is that works fine when a developer
is testing on a platform where `env` is available and behaves as
expected, but it then breaks on other platforms.

`env` calling `env` can make sense if one such `env` is within a lit
substitution, as in D65156 and D65121.  This patch ensures that lit
executes both as internal commands.

Reviewed By: probinson, mgorny, rnk

Differential Revision: https://reviews.llvm.org/D65697

llvm/utils/lit/lit/TestRunner.py

@@ -234,7 +234,9 @@ def quote_windows_command(seq):
     return ''.join(result)
 
 # args are from 'export' or 'env' command.
-# Returns copy of args without those commands or their arguments.
+# Skips the command, and parses its arguments.
+# Modifies env accordingly.
+# Returns copy of args without the command or its arguments.
 def updateEnv(env, args):
     arg_idx_next = len(args)
     unset_next_env_var = False
@@ -625,12 +627,16 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
         # Reference the global environment by default.
         cmd_shenv = shenv
         args = list(j.args)
-        if j.args[0] == 'env':
+        while args[0] == 'env':
             # Create a copy of the global environment and modify it for this one
-            # command. There might be multiple envs in a pipeline:
+            # command. There might be multiple envs in a pipeline, and
+            # there might be multiple envs in a command (usually when one comes
+            # from a substitution):
             #   env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
-            cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
-            args = updateEnv(cmd_shenv, j.args)
+            #   env FOO=1 %{another_env_plus_cmd} | FileCheck %s
+            if cmd_shenv is shenv:
+                cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
+            args = updateEnv(cmd_shenv, args)
             if not args:
                 raise InternalShellError(j,
                                          "Error: 'env' requires a subcommand")

llvm/utils/lit/tests/Inputs/shtest-env/env-args-nested-none.txt

@@ -0,0 +1 @@
+# RUN: env env env

llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt

@@ -0,0 +1,32 @@
+# Check that internal env can call internal env.
+
+# RUN: env env %{python} print_environment.py \
+# RUN: | FileCheck -check-prefix=CHECK-2-EMPTY %s
+#
+# CHECK-2-EMPTY: BAR = 2
+# CHECK-2-EMPTY: FOO = 1
+
+# RUN: env FOO=2 env BAR=1 %{python} print_environment.py \
+# RUN: | FileCheck -check-prefix=CHECK-2-VAL %s
+#
+# CHECK-2-VAL: BAR = 1
+# CHECK-2-VAL: FOO = 2
+
+# RUN: env -u FOO env -u BAR %{python} print_environment.py \
+# RUN: | FileCheck -check-prefix=CHECK-2-U %s
+#
+# CHECK-2-U-NOT: BAR
+# CHECK-2-U-NOT: FOO
+
+# RUN: env -u FOO BAR=1 env -u BAR FOO=2 %{python} print_environment.py \
+# RUN: | FileCheck -check-prefix=CHECK-2-U-VAL %s
+#
+# CHECK-2-U-VAL-NOT: BAR
+# CHECK-2-U-VAL: FOO = 2
+
+# RUN: env -u FOO BAR=1 env -u BAR FOO=2 env BAZ=3 %{python} print_environment.py \
+# RUN: | FileCheck -check-prefix=CHECK-3 %s
+#
+# CHECK-3-NOT: BAR
+# CHECK-3: BAZ = 3
+# CHECK-3: FOO = 2

llvm/utils/lit/tests/shtest-env.py

@@ -7,21 +7,30 @@
 
 # Make sure env commands are included in printed commands.
 
-# CHECK: -- Testing: 13 tests{{.*}}
+# CHECK: -- Testing: 15 tests{{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-assign.txt ({{[^)]*}})
+# CHECK: $ "env" "FOO=1"
 # CHECK: Error: 'env' requires a subcommand
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-u-arg.txt ({{[^)]*}})
+# CHECK: $ "env" "-u" "FOO"
 # CHECK: Error: 'env' requires a subcommand
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-u.txt ({{[^)]*}})
+# CHECK: $ "env" "-u"
+# CHECK: Error: 'env' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-env :: env-args-nested-none.txt ({{[^)]*}})
+# CHECK: $ "env" "env" "env"
 # CHECK: Error: 'env' requires a subcommand
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-none.txt ({{[^)]*}})
+# CHECK: $ "env"
 # CHECK: Error: 'env' requires a subcommand
 # CHECK: error: command failed with exit status: {{.*}}
 
@@ -40,6 +49,14 @@
 # CHECK: Error: 'env' cannot call 'echo'
 # CHECK: error: command failed with exit status: {{.*}}
 
+# CHECK: PASS: shtest-env :: env-calls-env.txt ({{[^)]*}})
+# CHECK: $ "env" "env" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "FOO=2" "env" "BAR=1" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "-u" "FOO" "env" "-u" "BAR" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "-u" "FOO" "BAR=1" "env" "-u" "BAR" "FOO=2" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "-u" "FOO" "BAR=1" "env" "-u" "BAR" "FOO=2" "env" "BAZ=3" "{{[^"]*}}" "print_environment.py"
+# CHECK-NOT: ${{.*}}print_environment.py
+
 # CHECK: FAIL: shtest-env :: env-calls-export.txt ({{[^)]*}})
 # CHECK: $ "env" "-u" "FOO" "BAR=3" "export" "BAZ=3"
 # CHECK: Error: 'env' cannot call 'export'
@@ -71,6 +88,6 @@
 # CHECK: $ "env" "A_FOO=1" "-u" "FOO" "B_BAR=2" "-u" "BAR" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
 # CHECK-NOT: ${{.*}}print_environment.py
 
-# CHECK: Expected Passes : 3
-# CHECK: Unexpected Failures: 10
+# CHECK: Expected Passes : 4
+# CHECK: Unexpected Failures: 11
 # CHECK-NOT: {{.}}