[lit] Implement non-pipelined ‘mkdir’, ‘diff’ and ‘rm’ commands internally

Summary:
The internal shell already supports 'cd', ‘export’ and ‘echo’ commands.
This patch adds implementation of non-pipelined ‘mkdir’, ‘diff’ and ‘rm’
commands as the internal shell builtins.

Reviewed by: Zachary Turner, Reid Kleckner

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D39567

llvm-svn: 319528
This commit is contained in:
Ying Yi 2017-12-01 09:54:27 +00:00
parent 196be4a18e
commit cfb08e9e6b
19 changed files with 456 additions and 2 deletions

View File

@ -1,7 +1,14 @@
from __future__ import absolute_import from __future__ import absolute_import
import difflib
import errno
import functools
import itertools
import getopt
import os, signal, subprocess, sys import os, signal, subprocess, sys
import re import re
import stat
import platform import platform
import shutil
import tempfile import tempfile
import threading import threading
@ -302,6 +309,152 @@ def executeBuiltinEcho(cmd, shenv):
return stdout.getvalue() return stdout.getvalue()
return "" return ""
def executeBuiltinMkdir(cmd, cmd_shenv):
"""executeBuiltinMkdir - Create new directories."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, 'p')
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'mkdir': %s" % str(err))
parent = False
for o, a in opts:
if o == "-p":
parent = True
else:
assert False, "unhandled option"
if len(args) == 0:
raise InternalShellError(cmd, "Error: 'mkdir' is missing an operand")
stderr = StringIO()
exitCode = 0
for dir in args:
if not os.path.isabs(dir):
dir = os.path.realpath(os.path.join(cmd_shenv.cwd, dir))
if parent:
lit.util.mkdir_p(dir)
else:
try:
os.mkdir(dir)
except OSError as err:
stderr.write("Error: 'mkdir' command failed, %s\n" % str(err))
exitCode = 1
return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
def executeBuiltinDiff(cmd, cmd_shenv):
"""executeBuiltinDiff - Compare files line by line."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, "wbu", ["strip-trailing-cr"])
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'diff': %s" % str(err))
filelines, filepaths = ([] for i in range(2))
ignore_all_space = False
ignore_space_change = False
unified_diff = False
strip_trailing_cr = False
for o, a in opts:
if o == "-w":
ignore_all_space = True
elif o == "-b":
ignore_space_change = True
elif o == "-u":
unified_diff = True
elif o == "--strip-trailing-cr":
strip_trailing_cr = True
else:
assert False, "unhandled option"
if len(args) != 2:
raise InternalShellError(cmd, "Error: missing or extra operand")
stderr = StringIO()
stdout = StringIO()
exitCode = 0
try:
for file in args:
if not os.path.isabs(file):
file = os.path.realpath(os.path.join(cmd_shenv.cwd, file))
filepaths.append(file)
with open(file, 'r') as f:
filelines.append(f.readlines())
def compose2(f, g):
return lambda x: f(g(x))
f = lambda x: x
if strip_trailing_cr:
f = compose2(lambda line: line.rstrip('\r'), f)
if ignore_all_space or ignore_space_change:
ignoreSpace = lambda line, separator: separator.join(line.split())
ignoreAllSpaceOrSpaceChange = functools.partial(ignoreSpace, separator='' if ignore_all_space else ' ')
f = compose2(ignoreAllSpaceOrSpaceChange, f)
for idx, lines in enumerate(filelines):
filelines[idx]= [f(line) for line in lines]
func = difflib.unified_diff if unified_diff else difflib.context_diff
for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1]):
stdout.write(diff)
exitCode = 1
except IOError as err:
stderr.write("Error: 'diff' command failed, %s\n" % str(err))
exitCode = 1
return ShellCommandResult(cmd, stdout.getvalue(), stderr.getvalue(), exitCode, False)
def executeBuiltinRm(cmd, cmd_shenv):
"""executeBuiltinRm - Removes (deletes) files or directories."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, "frR", ["--recursive"])
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'rm': %s" % str(err))
force = False
recursive = False
for o, a in opts:
if o == "-f":
force = True
elif o in ("-r", "-R", "--recursive"):
recursive = True
else:
assert False, "unhandled option"
if len(args) == 0:
raise InternalShellError(cmd, "Error: 'rm' is missing an operand")
def on_rm_error(func, path, exc_info):
# path contains the path of the file that couldn't be removed
# let's just assume that it's read-only and remove it.
os.chmod(path, stat.S_IMODE( os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
stderr = StringIO()
exitCode = 0
for path in args:
if not os.path.isabs(path):
path = os.path.realpath(os.path.join(cmd_shenv.cwd, path))
if force and not os.path.exists(path):
continue
try:
if os.path.isdir(path):
if not recursive:
stderr.write("Error: %s is a directory\n" % path)
exitCode = 1
shutil.rmtree(path, onerror = on_rm_error if force else None)
else:
if force and not os.access(path, os.W_OK):
os.chmod(path,
stat.S_IMODE(os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
except OSError as err:
stderr.write("Error: 'rm' command failed, %s" % str(err))
exitCode = 1
return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
def processRedirects(cmd, stdin_source, cmd_shenv, opened_files): def processRedirects(cmd, stdin_source, cmd_shenv, opened_files):
"""Return the standard fds for cmd after applying redirects """Return the standard fds for cmd after applying redirects
@ -460,6 +613,30 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
updateEnv(shenv, cmd.commands[0]) updateEnv(shenv, cmd.commands[0])
return 0 return 0
if cmd.commands[0].args[0] == 'mkdir':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'mkdir' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinMkdir(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
if cmd.commands[0].args[0] == 'diff':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'diff' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinDiff(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
if cmd.commands[0].args[0] == 'rm':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'rm' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinRm(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
procs = [] procs = []
default_stdin = subprocess.PIPE default_stdin = subprocess.PIPE
stderrTempFiles = [] stderrTempFiles = []

View File

@ -0,0 +1,28 @@
#!/usr/bin/env python
import os
import sys
def check_path(argv):
if len(argv) < 3:
print("Wrong number of args")
return 1
type = argv[1]
paths = argv[2:]
exit_code = 0
if type == 'dir':
for idx, dir in enumerate(paths):
print(os.path.isdir(dir))
elif type == 'file':
for idx, file in enumerate(paths):
print(os.path.isfile(file))
else:
print("Unrecognised type {}".format(type))
exit_code = 1
return exit_code
if __name__ == '__main__':
sys.exit (check_path (sys.argv))

View File

@ -0,0 +1,3 @@
# Check error on a unsupported diff (cannot be part of a pipeline).
#
# RUN: diff diff-error-0.txt diff-error-0.txt | echo Output

View File

@ -0,0 +1,3 @@
# Check error on a unsupported diff (not support the -B option).
#
# RUN: diff -B temp1.txt temp2.txt

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (missing tofile)
#
# RUN: diff temp.txt

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (unable to find compared files)
#
# RUN: diff temp.txt temp1.txt

View File

@ -0,0 +1,5 @@
# Check exit code error (file's contents are different)
#
# RUN: echo "hello-first" > %t
# RUN: echo "hello-second" > %t1
# RUN: diff %t %t1

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (missing operand)
#
# RUN: diff -u

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (extra operand)
#
# RUN: diff -u a.txt b.txt c.txt

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (cannot be part of a pipeline).
#
# RUN: mkdir -p temp | rm -rf temp

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (only does not support -m option).
#
# RUN: mkdir -p -m 777 temp

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (missing operand).
#
# RUN: mkdir -p

View File

@ -0,0 +1,3 @@
# Check error on a unsupported rm. (cannot be part of a pipeline)
#
# RUN: rm -rf temp | echo "hello"

View File

@ -0,0 +1,3 @@
# Check error on a unsupported rm (only does not support -v option).
#
# RUN: rm -f -v temp

View File

@ -0,0 +1,4 @@
# Check error on a unsupported rm (only supports -f option and in combination with -r).
#
#### RUN: rm -r hello
# RUN: rm hello

View File

@ -0,0 +1,4 @@
# Check error on a unsupported rm (can't remove test since it is a directory).
#
# RUN: mkdir -p test
# RUN: rm -f test

View File

@ -0,0 +1,87 @@
# Check rm file operations.
# Check force remove commands success whether the file does or doesn't exist.
#
# RUN: rm -f %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=REMOVE-FILE < %t.out %s
# RUN: echo "create a temp file" > %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=FILE-EXIST < %t.out %s
# RUN: rm -f %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=REMOVE-FILE < %t.out %s
#
# REMOVE-FILE: False
# FILE-EXIST: True
#
# Check mkdir and rm folder operations.
# Check force remove commands success whether the directory does or doesn't exist.
#
# Check the mkdir command with -p option.
# RUN: rm -f -r %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-PARENT-DIR < %t.out %s
# RUN: mkdir -p %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=MAKE-PARENT-DIR < %t.out %s
# RUN: rm -f %T/test || true
# RUN: rm -f -r %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-PARENT-DIR < %t.out %s
#
# MAKE-PARENT-DIR: True
# REMOVE-PARENT-DIR: False
#
# Check the mkdir command without -p option.
#
# RUN: rm -rf %T/test1
# RUN: mkdir %T/test1
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=MAKE-DIR < %t.out %s
# RUN: cd %T/test1 && mkdir foo
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=MAKE-DIR < %t.out %s
# RUN: cd %T && rm -rf %T/test1
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=REMOVE-DIR < %t.out %s
#
# MAKE-DIR: True
# REMOVE-DIR: False
#
# Check creating and removing multiple folders and rm * operation.
#
# RUN: rm -rf %T/test
# RUN: mkdir -p %T/test/test1 %T/test/test2
# RUN: "%{python}" %S/check_path.py dir %T/test %T/test/test1 %T/test/test2 > %t.out
# RUN: FileCheck --check-prefix=DIRS-EXIST < %t.out %s
# RUN: mkdir %T/test || true
# RUN: echo "create a temp file" > %T/test/temp.write
# RUN: echo "create a temp1 file" > %T/test/test1/temp1.write
# RUN: echo "create a temp2 file" > %T/test/test2/temp2.write
# RUN: "%{python}" %S/check_path.py file %T/test/temp.write %T/test/test1/temp1.write %T/test/test2/temp2.write> %t.out
# RUN: FileCheck --check-prefix=FILES-EXIST < %t.out %s
# RUN: rm -r -f %T/*
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-ALL < %t.out %s
#
# DIRS-EXIST: True
# DIRS-EXIST-NEXT: True
# DIRS-EXIST-NEXT: True
# FILES-EXIST: True
# FILES-EXIST-NEXT: True
# FILES-EXIST-NEXT: True
# REMOVE-ALL: False
#
# Check diff operations.
#
# RUN: echo "hello" > %t.stdout
# RUN: echo "hello" > %t1.stdout
# RUN: diff %t.stdout %t1.stdout
# RUN: diff -u %t.stdout %t1.stdout
# RUN: echo "hello-2" > %t1.stdout
# RUN: diff %t.stdout %t1.stdout || true
#
# RUN: mkdir -p %T/dir1 %T/dir2
# RUN: cd %T/dir1 && echo "hello" > temp1.txt
# RUN: cd %T/dir2 && echo "hello" > temp2.txt
# RUN: diff temp2.txt ../dir1/temp1.txt

View File

@ -8,7 +8,7 @@
# #
# END. # END.
# CHECK: Failing Tests (3) # CHECK: Failing Tests (17)
# CHECK: Failing Tests (1) # CHECK: Failing Tests (1)
# CHECK: Failing Tests (2) # CHECK: Failing Tests (2)
# CHECK: error: Setting --max-failures to 0 does not have any effect. # CHECK: error: Setting --max-failures to 0 does not have any effect.

View File

@ -10,6 +10,67 @@
# CHECK: -- Testing: # CHECK: -- Testing:
# CHECK: FAIL: shtest-shell :: diff-error-0.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
# CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
# CHECK: # command stderr:
# CHECK: Unsupported: 'diff' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-1.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-1.txt' FAILED ***
# CHECK: $ "diff" "-B" "temp1.txt" "temp2.txt"
# CHECK: # command stderr:
# CHECK: Unsupported: 'diff': option -B not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-2.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-2.txt' FAILED ***
# CHECK: $ "diff" "temp.txt"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-3.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-3.txt' FAILED ***
# CHECK: $ "diff" "temp.txt" "temp1.txt"
# CHECK: # command stderr:
# CHECK: Error: 'diff' command failed
# CHECK: error: command failed with exit status: 1
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-4.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-4.txt' FAILED ***
# CHECK: Exit Code: 1
# CHECK: # command output:
# CHECK: diff-error-4.txt.tmp
# CHECK: diff-error-4.txt.tmp1
# CHECK: *** 1 ****
# CHECK: ! hello-first
# CHECK: --- 1 ----
# CHECK: ! hello-second
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-5.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-5.txt' FAILED ***
# CHECK: $ "diff"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-6.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-6.txt' FAILED ***
# CHECK: $ "diff"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: error-0.txt # CHECK: FAIL: shtest-shell :: error-0.txt
# CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED *** # CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
# CHECK: $ "not-a-real-command" # CHECK: $ "not-a-real-command"
@ -30,7 +91,62 @@
# CHECK: Unsupported redirect: # CHECK: Unsupported redirect:
# CHECK: *** # CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-0.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-0.txt' FAILED ***
# CHECK: $ "mkdir" "-p" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'mkdir' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-1.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-1.txt' FAILED ***
# CHECK: $ "mkdir" "-p" "-m" "777" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'mkdir': option -m not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-2.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-2.txt' FAILED ***
# CHECK: $ "mkdir" "-p"
# CHECK: # command stderr:
# CHECK: Error: 'mkdir' is missing an operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: PASS: shtest-shell :: redirects.txt # CHECK: PASS: shtest-shell :: redirects.txt
# CHECK: FAIL: shtest-shell :: rm-error-0.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-0.txt' FAILED ***
# CHECK: $ "rm" "-rf" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'rm' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-1.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-1.txt' FAILED ***
# CHECK: $ "rm" "-f" "-v" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'rm': option -v not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-2.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-2.txt' FAILED ***
# CHECK: $ "rm" "-r" "hello"
# CHECK: # command stderr:
# CHECK: Error: 'rm' command failed
# CHECK: error: command failed with exit status: 1
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-3.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-3.txt' FAILED ***
# CHECK: Exit Code: 1
# CHECK: ***
# CHECK: PASS: shtest-shell :: sequencing-0.txt # CHECK: PASS: shtest-shell :: sequencing-0.txt
# CHECK: XFAIL: shtest-shell :: sequencing-1.txt # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
# CHECK: Failing Tests (3) # CHECK: PASS: shtest-shell :: valid-shell.txt
# CHECK: Failing Tests (17)