[lit] Implement non-pipelined ‘mkdir’, ‘diff’ and ‘rm’ commands internally

Summary:
The internal shell already supports 'cd', ‘export’ and ‘echo’ commands.
This patch adds implementation of non-pipelined ‘mkdir’, ‘diff’ and ‘rm’
commands as the internal shell builtins.

Reviewed by: Zachary Turner, Reid Kleckner

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D39567

llvm-svn: 319528
This commit is contained in:
Ying Yi 2017-12-01 09:54:27 +00:00
parent 196be4a18e
commit cfb08e9e6b
19 changed files with 456 additions and 2 deletions

View File

@ -1,7 +1,14 @@
from __future__ import absolute_import
import difflib
import errno
import functools
import itertools
import getopt
import os, signal, subprocess, sys
import re
import stat
import platform
import shutil
import tempfile
import threading
@ -302,6 +309,152 @@ def executeBuiltinEcho(cmd, shenv):
return stdout.getvalue()
return ""
def executeBuiltinMkdir(cmd, cmd_shenv):
"""executeBuiltinMkdir - Create new directories."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, 'p')
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'mkdir': %s" % str(err))
parent = False
for o, a in opts:
if o == "-p":
parent = True
else:
assert False, "unhandled option"
if len(args) == 0:
raise InternalShellError(cmd, "Error: 'mkdir' is missing an operand")
stderr = StringIO()
exitCode = 0
for dir in args:
if not os.path.isabs(dir):
dir = os.path.realpath(os.path.join(cmd_shenv.cwd, dir))
if parent:
lit.util.mkdir_p(dir)
else:
try:
os.mkdir(dir)
except OSError as err:
stderr.write("Error: 'mkdir' command failed, %s\n" % str(err))
exitCode = 1
return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
def executeBuiltinDiff(cmd, cmd_shenv):
"""executeBuiltinDiff - Compare files line by line."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, "wbu", ["strip-trailing-cr"])
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'diff': %s" % str(err))
filelines, filepaths = ([] for i in range(2))
ignore_all_space = False
ignore_space_change = False
unified_diff = False
strip_trailing_cr = False
for o, a in opts:
if o == "-w":
ignore_all_space = True
elif o == "-b":
ignore_space_change = True
elif o == "-u":
unified_diff = True
elif o == "--strip-trailing-cr":
strip_trailing_cr = True
else:
assert False, "unhandled option"
if len(args) != 2:
raise InternalShellError(cmd, "Error: missing or extra operand")
stderr = StringIO()
stdout = StringIO()
exitCode = 0
try:
for file in args:
if not os.path.isabs(file):
file = os.path.realpath(os.path.join(cmd_shenv.cwd, file))
filepaths.append(file)
with open(file, 'r') as f:
filelines.append(f.readlines())
def compose2(f, g):
return lambda x: f(g(x))
f = lambda x: x
if strip_trailing_cr:
f = compose2(lambda line: line.rstrip('\r'), f)
if ignore_all_space or ignore_space_change:
ignoreSpace = lambda line, separator: separator.join(line.split())
ignoreAllSpaceOrSpaceChange = functools.partial(ignoreSpace, separator='' if ignore_all_space else ' ')
f = compose2(ignoreAllSpaceOrSpaceChange, f)
for idx, lines in enumerate(filelines):
filelines[idx]= [f(line) for line in lines]
func = difflib.unified_diff if unified_diff else difflib.context_diff
for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1]):
stdout.write(diff)
exitCode = 1
except IOError as err:
stderr.write("Error: 'diff' command failed, %s\n" % str(err))
exitCode = 1
return ShellCommandResult(cmd, stdout.getvalue(), stderr.getvalue(), exitCode, False)
def executeBuiltinRm(cmd, cmd_shenv):
"""executeBuiltinRm - Removes (deletes) files or directories."""
args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
opts, args = getopt.gnu_getopt(args, "frR", ["--recursive"])
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'rm': %s" % str(err))
force = False
recursive = False
for o, a in opts:
if o == "-f":
force = True
elif o in ("-r", "-R", "--recursive"):
recursive = True
else:
assert False, "unhandled option"
if len(args) == 0:
raise InternalShellError(cmd, "Error: 'rm' is missing an operand")
def on_rm_error(func, path, exc_info):
# path contains the path of the file that couldn't be removed
# let's just assume that it's read-only and remove it.
os.chmod(path, stat.S_IMODE( os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
stderr = StringIO()
exitCode = 0
for path in args:
if not os.path.isabs(path):
path = os.path.realpath(os.path.join(cmd_shenv.cwd, path))
if force and not os.path.exists(path):
continue
try:
if os.path.isdir(path):
if not recursive:
stderr.write("Error: %s is a directory\n" % path)
exitCode = 1
shutil.rmtree(path, onerror = on_rm_error if force else None)
else:
if force and not os.access(path, os.W_OK):
os.chmod(path,
stat.S_IMODE(os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
except OSError as err:
stderr.write("Error: 'rm' command failed, %s" % str(err))
exitCode = 1
return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
def processRedirects(cmd, stdin_source, cmd_shenv, opened_files):
"""Return the standard fds for cmd after applying redirects
@ -460,6 +613,30 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
updateEnv(shenv, cmd.commands[0])
return 0
if cmd.commands[0].args[0] == 'mkdir':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'mkdir' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinMkdir(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
if cmd.commands[0].args[0] == 'diff':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'diff' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinDiff(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
if cmd.commands[0].args[0] == 'rm':
if len(cmd.commands) != 1:
raise InternalShellError(cmd.commands[0], "Unsupported: 'rm' "
"cannot be part of a pipeline")
cmdResult = executeBuiltinRm(cmd.commands[0], shenv)
results.append(cmdResult)
return cmdResult.exitCode
procs = []
default_stdin = subprocess.PIPE
stderrTempFiles = []

View File

@ -0,0 +1,28 @@
#!/usr/bin/env python
import os
import sys
def check_path(argv):
if len(argv) < 3:
print("Wrong number of args")
return 1
type = argv[1]
paths = argv[2:]
exit_code = 0
if type == 'dir':
for idx, dir in enumerate(paths):
print(os.path.isdir(dir))
elif type == 'file':
for idx, file in enumerate(paths):
print(os.path.isfile(file))
else:
print("Unrecognised type {}".format(type))
exit_code = 1
return exit_code
if __name__ == '__main__':
sys.exit (check_path (sys.argv))

View File

@ -0,0 +1,3 @@
# Check error on a unsupported diff (cannot be part of a pipeline).
#
# RUN: diff diff-error-0.txt diff-error-0.txt | echo Output

View File

@ -0,0 +1,3 @@
# Check error on a unsupported diff (not support the -B option).
#
# RUN: diff -B temp1.txt temp2.txt

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (missing tofile)
#
# RUN: diff temp.txt

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (unable to find compared files)
#
# RUN: diff temp.txt temp1.txt

View File

@ -0,0 +1,5 @@
# Check exit code error (file's contents are different)
#
# RUN: echo "hello-first" > %t
# RUN: echo "hello-second" > %t1
# RUN: diff %t %t1

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (missing operand)
#
# RUN: diff -u

View File

@ -0,0 +1,3 @@
# Check error on an internal shell error (extra operand)
#
# RUN: diff -u a.txt b.txt c.txt

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (cannot be part of a pipeline).
#
# RUN: mkdir -p temp | rm -rf temp

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (only does not support -m option).
#
# RUN: mkdir -p -m 777 temp

View File

@ -0,0 +1,3 @@
# Check error on a unsupported mkdir (missing operand).
#
# RUN: mkdir -p

View File

@ -0,0 +1,3 @@
# Check error on a unsupported rm. (cannot be part of a pipeline)
#
# RUN: rm -rf temp | echo "hello"

View File

@ -0,0 +1,3 @@
# Check error on a unsupported rm (only does not support -v option).
#
# RUN: rm -f -v temp

View File

@ -0,0 +1,4 @@
# Check error on a unsupported rm (only supports -f option and in combination with -r).
#
#### RUN: rm -r hello
# RUN: rm hello

View File

@ -0,0 +1,4 @@
# Check error on a unsupported rm (can't remove test since it is a directory).
#
# RUN: mkdir -p test
# RUN: rm -f test

View File

@ -0,0 +1,87 @@
# Check rm file operations.
# Check force remove commands success whether the file does or doesn't exist.
#
# RUN: rm -f %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=REMOVE-FILE < %t.out %s
# RUN: echo "create a temp file" > %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=FILE-EXIST < %t.out %s
# RUN: rm -f %t.write
# RUN: "%{python}" %S/check_path.py file %t.write > %t.out
# RUN: FileCheck --check-prefix=REMOVE-FILE < %t.out %s
#
# REMOVE-FILE: False
# FILE-EXIST: True
#
# Check mkdir and rm folder operations.
# Check force remove commands success whether the directory does or doesn't exist.
#
# Check the mkdir command with -p option.
# RUN: rm -f -r %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-PARENT-DIR < %t.out %s
# RUN: mkdir -p %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=MAKE-PARENT-DIR < %t.out %s
# RUN: rm -f %T/test || true
# RUN: rm -f -r %T/test
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-PARENT-DIR < %t.out %s
#
# MAKE-PARENT-DIR: True
# REMOVE-PARENT-DIR: False
#
# Check the mkdir command without -p option.
#
# RUN: rm -rf %T/test1
# RUN: mkdir %T/test1
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=MAKE-DIR < %t.out %s
# RUN: cd %T/test1 && mkdir foo
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=MAKE-DIR < %t.out %s
# RUN: cd %T && rm -rf %T/test1
# RUN: "%{python}" %S/check_path.py dir %T/test1 > %t.out
# RUN: FileCheck --check-prefix=REMOVE-DIR < %t.out %s
#
# MAKE-DIR: True
# REMOVE-DIR: False
#
# Check creating and removing multiple folders and rm * operation.
#
# RUN: rm -rf %T/test
# RUN: mkdir -p %T/test/test1 %T/test/test2
# RUN: "%{python}" %S/check_path.py dir %T/test %T/test/test1 %T/test/test2 > %t.out
# RUN: FileCheck --check-prefix=DIRS-EXIST < %t.out %s
# RUN: mkdir %T/test || true
# RUN: echo "create a temp file" > %T/test/temp.write
# RUN: echo "create a temp1 file" > %T/test/test1/temp1.write
# RUN: echo "create a temp2 file" > %T/test/test2/temp2.write
# RUN: "%{python}" %S/check_path.py file %T/test/temp.write %T/test/test1/temp1.write %T/test/test2/temp2.write> %t.out
# RUN: FileCheck --check-prefix=FILES-EXIST < %t.out %s
# RUN: rm -r -f %T/*
# RUN: "%{python}" %S/check_path.py dir %T/test > %t.out
# RUN: FileCheck --check-prefix=REMOVE-ALL < %t.out %s
#
# DIRS-EXIST: True
# DIRS-EXIST-NEXT: True
# DIRS-EXIST-NEXT: True
# FILES-EXIST: True
# FILES-EXIST-NEXT: True
# FILES-EXIST-NEXT: True
# REMOVE-ALL: False
#
# Check diff operations.
#
# RUN: echo "hello" > %t.stdout
# RUN: echo "hello" > %t1.stdout
# RUN: diff %t.stdout %t1.stdout
# RUN: diff -u %t.stdout %t1.stdout
# RUN: echo "hello-2" > %t1.stdout
# RUN: diff %t.stdout %t1.stdout || true
#
# RUN: mkdir -p %T/dir1 %T/dir2
# RUN: cd %T/dir1 && echo "hello" > temp1.txt
# RUN: cd %T/dir2 && echo "hello" > temp2.txt
# RUN: diff temp2.txt ../dir1/temp1.txt

View File

@ -8,7 +8,7 @@
#
# END.
# CHECK: Failing Tests (3)
# CHECK: Failing Tests (17)
# CHECK: Failing Tests (1)
# CHECK: Failing Tests (2)
# CHECK: error: Setting --max-failures to 0 does not have any effect.

View File

@ -10,6 +10,67 @@
# CHECK: -- Testing:
# CHECK: FAIL: shtest-shell :: diff-error-0.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
# CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
# CHECK: # command stderr:
# CHECK: Unsupported: 'diff' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-1.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-1.txt' FAILED ***
# CHECK: $ "diff" "-B" "temp1.txt" "temp2.txt"
# CHECK: # command stderr:
# CHECK: Unsupported: 'diff': option -B not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-2.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-2.txt' FAILED ***
# CHECK: $ "diff" "temp.txt"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-3.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-3.txt' FAILED ***
# CHECK: $ "diff" "temp.txt" "temp1.txt"
# CHECK: # command stderr:
# CHECK: Error: 'diff' command failed
# CHECK: error: command failed with exit status: 1
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-4.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-4.txt' FAILED ***
# CHECK: Exit Code: 1
# CHECK: # command output:
# CHECK: diff-error-4.txt.tmp
# CHECK: diff-error-4.txt.tmp1
# CHECK: *** 1 ****
# CHECK: ! hello-first
# CHECK: --- 1 ----
# CHECK: ! hello-second
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-5.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-5.txt' FAILED ***
# CHECK: $ "diff"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: diff-error-6.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-6.txt' FAILED ***
# CHECK: $ "diff"
# CHECK: # command stderr:
# CHECK: Error: missing or extra operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: error-0.txt
# CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
# CHECK: $ "not-a-real-command"
@ -30,7 +91,62 @@
# CHECK: Unsupported redirect:
# CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-0.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-0.txt' FAILED ***
# CHECK: $ "mkdir" "-p" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'mkdir' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-1.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-1.txt' FAILED ***
# CHECK: $ "mkdir" "-p" "-m" "777" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'mkdir': option -m not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: mkdir-error-2.txt
# CHECK: *** TEST 'shtest-shell :: mkdir-error-2.txt' FAILED ***
# CHECK: $ "mkdir" "-p"
# CHECK: # command stderr:
# CHECK: Error: 'mkdir' is missing an operand
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: PASS: shtest-shell :: redirects.txt
# CHECK: FAIL: shtest-shell :: rm-error-0.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-0.txt' FAILED ***
# CHECK: $ "rm" "-rf" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'rm' cannot be part of a pipeline
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-1.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-1.txt' FAILED ***
# CHECK: $ "rm" "-f" "-v" "temp"
# CHECK: # command stderr:
# CHECK: Unsupported: 'rm': option -v not recognized
# CHECK: error: command failed with exit status: 127
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-2.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-2.txt' FAILED ***
# CHECK: $ "rm" "-r" "hello"
# CHECK: # command stderr:
# CHECK: Error: 'rm' command failed
# CHECK: error: command failed with exit status: 1
# CHECK: ***
# CHECK: FAIL: shtest-shell :: rm-error-3.txt
# CHECK: *** TEST 'shtest-shell :: rm-error-3.txt' FAILED ***
# CHECK: Exit Code: 1
# CHECK: ***
# CHECK: PASS: shtest-shell :: sequencing-0.txt
# CHECK: XFAIL: shtest-shell :: sequencing-1.txt
# CHECK: Failing Tests (3)
# CHECK: PASS: shtest-shell :: valid-shell.txt
# CHECK: Failing Tests (17)