maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Move libcxx/test/libcxx python package into libcxx/utils/libcxx. 

This patch merges the test python package with the newly
created package in utils.

llvm-svn: 294651
This commit is contained in:
Eric Fiselier 2017-02-09 23:18:11 +00:00
parent f38bab73aa
commit 4f258efad7
13 changed files with 310 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
libcxx/test/libcxx/test/__init__.py
View File

55
libcxx/test/libcxx/util.py
View File

@ -1,55 +0,0 @@
#===----------------------------------------------------------------------===##
#
# The LLVM Compiler Infrastructure
#
# This file is dual licensed under the MIT and the University of Illinois Open
# Source Licenses. See LICENSE.TXT for details.
#
#===----------------------------------------------------------------------===##
from contextlib import contextmanager
import os
import tempfile
def cleanFile(filename):
try:
os.remove(filename)
except OSError:
pass
@contextmanager
def guardedTempFilename(suffix='', prefix='', dir=None):
# Creates and yeilds a temporary filename within a with statement. The file
# is removed upon scope exit.
handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
os.close(handle)
yield name
cleanFile(name)
@contextmanager
def guardedFilename(name):
# yeilds a filename within a with statement. The file is removed upon scope
# exit.
yield name
cleanFile(name)
@contextmanager
def nullContext(value):
# yeilds a variable within a with statement. No action is taken upon scope
# exit.
yield value
def makeReport(cmd, out, err, rc):
report = "Command: %s\n" % cmd
report += "Exit Code: %d\n" % rc
if out:
report += "Standard Output:\n--\n%s--\n" % out
if err:
report += "Standard Error:\n--\n%s--\n" % err
report += '\n'
return report

2
libcxx/test/lit.cfg
View File

@ -3,7 +3,7 @@
import os
import site
site.addsitedir(os.path.dirname(__file__))
site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'utils'))
# Tell pylint that we know config and lit_config exist somewhere.

23
libcxx/test/libcxx/compiler.py → libcxx/utils/libcxx/compiler.py
View File

@ -9,7 +9,6 @@
import platform
import os
import lit.util
import libcxx.util
@ -154,27 +153,27 @@ class CXXCompiler(object):
def preprocess(self, source_files, out=None, flags=[], cwd=None):
cmd = self.preprocessCmd(source_files, out, flags)
out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
return cmd, out, err, rc
def compile(self, source_files, out=None, flags=[], cwd=None):
cmd = self.compileCmd(source_files, out, flags)
out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
return cmd, out, err, rc
def link(self, source_files, out=None, flags=[], cwd=None):
cmd = self.linkCmd(source_files, out, flags)
out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
return cmd, out, err, rc
def compileLink(self, source_files, out=None, flags=[],
cwd=None):
cmd = self.compileLinkCmd(source_files, out, flags)
out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
cwd=cwd)
return cmd, out, err, rc
def compileLinkTwoSteps(self, source_file, out=None, object_file=None,
@ -216,7 +215,7 @@ class CXXCompiler(object):
def getTriple(self):
cmd = [self.path] + self.flags + ['-dumpmachine']
return lit.util.capture(cmd).strip()
return libcxx.util.capture(cmd).strip()
def hasCompileFlag(self, flag):
if isinstance(flag, list):
@ -275,8 +274,8 @@ class CXXCompiler(object):
# TODO(EricWF): Are there other flags we need to worry about?
if '-v' in cmd:
cmd.remove('-v')
out, err, rc = lit.util.executeCommand(
cmd, input=lit.util.to_bytes('#error\n'))
out, err, rc = libcxx.util.executeCommand(
cmd, input=libcxx.util.to_bytes('#error\n'))
assert rc != 0
if flag in err:

5
libcxx/utils/libcxx/sym_check/extract.py
View File

@ -14,6 +14,7 @@ import distutils.spawn
import sys
import re
import libcxx.util
from libcxx.sym_check import util
extract_ignore_names = ['_init', '_fini']
@ -48,7 +49,7 @@ class NMExtractor(object):
parsed symbols.
"""
cmd = [self.nm_exe] + self.flags + [lib]
out, _, exit_code = util.execute_command_verbose(cmd)
out, _, exit_code = util.executeCommandVerbose(cmd)
if exit_code != 0:
raise RuntimeError('Failed to run %s on %s' % (self.nm_exe, lib))
fmt_syms = (self._extract_sym(l)
@ -134,7 +135,7 @@ class ReadElfExtractor(object):
parsed symbols.
"""
cmd = [self.tool] + self.flags + [lib]
out, _, exit_code = util.execute_command_verbose(cmd)
out, _, exit_code = libcxx.util.executeCommandVerbose(cmd)
if exit_code != 0:
raise RuntimeError('Failed to run %s on %s' % (self.nm_exe, lib))
dyn_syms = self.get_dynsym_table(out)

61
libcxx/utils/libcxx/sym_check/util.py
View File

@ -9,64 +9,9 @@
import ast
import distutils.spawn
import signal
import subprocess
import sys
import re
def to_bytes(str):
# Encode to UTF-8 to get binary data.
if isinstance(str, bytes):
return str
return str.encode('utf-8')
def to_string(bytes):
if isinstance(bytes, str):
return bytes
return to_bytes(bytes)
def convert_string(bytes):
try:
return to_string(bytes.decode('utf-8'))
except AttributeError: # 'str' object has no attribute 'decode'.
return str(bytes)
except UnicodeError:
return str(bytes)
def execute_command(cmd, input_str=None):
"""
Execute a command, capture and return its output.
"""
kwargs = {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
p = subprocess.Popen(cmd, **kwargs)
out, err = p.communicate(input=input_str)
exitCode = p.wait()
if exitCode == -signal.SIGINT:
raise KeyboardInterrupt
out = convert_string(out)
err = convert_string(err)
return out, err, exitCode
def execute_command_verbose(cmd, input_str=None):
"""
Execute a command and print its output on failure.
"""
out, err, exitCode = execute_command(cmd, input_str=input_str)
if exitCode != 0:
report = "Command: %s\n" % ' '.join(["'%s'" % a for a in cmd])
report += "Exit Code: %d\n" % exitCode
if out:
report += "Standard Output:\n--\n%s--" % out
if err:
report += "Standard Error:\n--\n%s--" % err
report += "\n\nFailed!"
sys.stderr.write('%s\n' % report)
return out, err, exitCode
import libcxx.util
def read_syms_from_list(slist):
@ -118,8 +63,8 @@ _cppfilt_exe = distutils.spawn.find_executable('c++filt')
def demangle_symbol(symbol):
if _cppfilt_exe is None:
return symbol
out, _, exit_code = execute_command_verbose(
[_cppfilt_exe], input_str=symbol)
out, _, exit_code = libcxx.util.executeCommandVerbose(
[_cppfilt_exe], input=symbol)
if exit_code != 0:
return symbol
return out

0
libcxx/test/libcxx/__init__.py → libcxx/utils/libcxx/test/__init__.py
View File

13
libcxx/test/libcxx/test/config.py → libcxx/utils/libcxx/test/config.py
View File

@ -17,10 +17,6 @@ import shlex
import shutil
import sys
import lit.Test # pylint: disable=import-error,no-name-in-module
import lit.util # pylint: disable=import-error,no-name-in-module
from libcxx.test.format import LibcxxTestFormat
from libcxx.compiler import CXXCompiler
from libcxx.test.target_info import make_target_info
from libcxx.test.executor import *
@ -164,6 +160,7 @@ class Configuration(object):
sys.stderr.flush() # Force flushing to avoid broken output on Windows
def get_test_format(self):
from libcxx.test.format import LibcxxTestFormat
return LibcxxTestFormat(
self.cxx,
self.use_clang_verify,
@ -200,7 +197,7 @@ class Configuration(object):
# If no specific cxx_under_test was given, attempt to infer it as
# clang++.
if cxx is None or self.cxx_is_clang_cl:
clangxx = lit.util.which('clang++',
clangxx = libcxx.util.which('clang++',
self.config.environment['PATH'])
if clangxx:
cxx = clangxx
@ -799,13 +796,13 @@ class Configuration(object):
# Search for llvm-symbolizer along the compiler path first
# and then along the PATH env variable.
symbolizer_search_paths = os.environ.get('PATH', '')
cxx_path = lit.util.which(self.cxx.path)
cxx_path = libcxx.util.which(self.cxx.path)
if cxx_path is not None:
symbolizer_search_paths = (
os.path.dirname(cxx_path) +
os.pathsep + symbolizer_search_paths)
llvm_symbolizer = lit.util.which('llvm-symbolizer',
symbolizer_search_paths)
llvm_symbolizer = libcxx.util.which('llvm-symbolizer',
symbolizer_search_paths)
def add_ubsan():
self.cxx.flags += ['-fsanitize=undefined',

3
libcxx/test/libcxx/test/executor.py → libcxx/utils/libcxx/test/executor.py
View File

@ -11,8 +11,7 @@ import platform
import os
from libcxx.test import tracing
from lit.util import executeCommand # pylint: disable=import-error
from libcxx.util import executeCommand
class Executor(object):

4
libcxx/test/libcxx/test/format.py → libcxx/utils/libcxx/test/format.py
View File

@ -17,8 +17,6 @@ import lit.Test # pylint: disable=import-error
import lit.TestRunner # pylint: disable=import-error
from lit.TestRunner import ParserKind, IntegratedTestKeywordParser \
# pylint: disable=import-error
import lit.util # pylint: disable=import-error
from libcxx.test.executor import LocalExecutor as LocalExecutor
import libcxx.util
@ -164,7 +162,7 @@ class LibcxxTestFormat(object):
exec_path = tmpBase + '.exe'
object_path = tmpBase + '.o'
# Create the output directory if it does not already exist.
lit.util.mkdir_p(os.path.dirname(tmpBase))
libcxx.util.mkdir_p(os.path.dirname(tmpBase))
try:
# Compile the test
cmd, out, err, rc = test_cxx.compileLinkTwoSteps(

1
libcxx/test/libcxx/test/target_info.py → libcxx/utils/libcxx/test/target_info.py
View File

@ -221,4 +221,3 @@ def make_target_info(full_config):
if target_system == 'Linux': return LinuxLocalTI(full_config)
if target_system == 'Windows': return WindowsLocalTI(full_config)
return DefaultTargetInfo(full_config)

0
libcxx/test/libcxx/test/tracing.py → libcxx/utils/libcxx/test/tracing.py
View File

285
libcxx/utils/libcxx/util.py Normal file
View File

@ -0,0 +1,285 @@
#===----------------------------------------------------------------------===##
#
# The LLVM Compiler Infrastructure
#
# This file is dual licensed under the MIT and the University of Illinois Open
# Source Licenses. See LICENSE.TXT for details.
#
#===----------------------------------------------------------------------===##
from contextlib import contextmanager
import os
import platform
import signal
import subprocess
import sys
import tempfile
# FIXME: Most of these functions are cribbed from LIT
def to_bytes(str):
# Encode to UTF-8 to get binary data.
if isinstance(str, bytes):
return str
return str.encode('utf-8')
def to_string(bytes):
if isinstance(bytes, str):
return bytes
return to_bytes(bytes)
def convert_string(bytes):
try:
return to_string(bytes.decode('utf-8'))
except AttributeError: # 'str' object has no attribute 'decode'.
return str(bytes)
except UnicodeError:
return str(bytes)
def cleanFile(filename):
try:
os.remove(filename)
except OSError:
pass
@contextmanager
def guardedTempFilename(suffix='', prefix='', dir=None):
# Creates and yeilds a temporary filename within a with statement. The file
# is removed upon scope exit.
handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
os.close(handle)
yield name
cleanFile(name)
@contextmanager
def guardedFilename(name):
# yeilds a filename within a with statement. The file is removed upon scope
# exit.
yield name
cleanFile(name)
@contextmanager
def nullContext(value):
# yeilds a variable within a with statement. No action is taken upon scope
# exit.
yield value
def makeReport(cmd, out, err, rc):
report = "Command: %s\n" % cmd
report += "Exit Code: %d\n" % rc
if out:
report += "Standard Output:\n--\n%s--\n" % out
if err:
report += "Standard Error:\n--\n%s--\n" % err
report += '\n'
return report
def capture(args, env=None):
"""capture(command) - Run the given command (or argv list) in a shell and
return the standard output. Raises a CalledProcessError if the command
exits with a non-zero status."""
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=env)
out, err = p.communicate()
out = convert_string(out)
err = convert_string(err)
if p.returncode != 0:
raise subprocess.CalledProcessError(cmd=args,
returncode=p.returncode,
output="{}\n{}".format(out, err))
return out
def which(command, paths = None):
"""which(command, [paths]) - Look up the given command in the paths string
(or the PATH environment variable, if unspecified)."""
if paths is None:
paths = os.environ.get('PATH','')
# Check for absolute match first.
if os.path.isfile(command):
return command
# Would be nice if Python had a lib function for this.
if not paths:
paths = os.defpath
# Get suffixes to search.
# On Cygwin, 'PATHEXT' may exist but it should not be used.
if os.pathsep == ';':
pathext = os.environ.get('PATHEXT', '').split(';')
else:
pathext = ['']
# Search the paths...
for path in paths.split(os.pathsep):
for ext in pathext:
p = os.path.join(path, command + ext)
if os.path.exists(p) and not os.path.isdir(p):
return p
return None
def checkToolsPath(dir, tools):
for tool in tools:
if not os.path.exists(os.path.join(dir, tool)):
return False
return True
def whichTools(tools, paths):
for path in paths.split(os.pathsep):
if checkToolsPath(path, tools):
return path
return None
def mkdir_p(path):
"""mkdir_p(path) - Make the "path" directory, if it does not exist; this
will also make directories for any missing parent directories."""
if not path or os.path.exists(path):
return
parent = os.path.dirname(path)
if parent != path:
mkdir_p(parent)
try:
os.mkdir(path)
except OSError:
e = sys.exc_info()[1]
# Ignore EEXIST, which may occur during a race condition.
if e.errno != errno.EEXIST:
raise
class ExecuteCommandTimeoutException(Exception):
def __init__(self, msg, out, err, exitCode):
assert isinstance(msg, str)
assert isinstance(out, str)
assert isinstance(err, str)
assert isinstance(exitCode, int)
self.msg = msg
self.out = out
self.err = err
self.exitCode = exitCode
# Close extra file handles on UNIX (on Windows this cannot be done while
# also redirecting input).
kUseCloseFDs = not (platform.system() == 'Windows')
def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
"""
Execute command ``command`` (list of arguments or string)
with
* working directory ``cwd`` (str), use None to use the current
working directory
* environment ``env`` (dict), use None for none
* Input to the command ``input`` (str), use string to pass
no input.
* Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
Returns a tuple (out, err, exitCode) where
* ``out`` (str) is the standard output of running the command
* ``err`` (str) is the standard error of running the command
* ``exitCode`` (int) is the exitCode of running the command
If the timeout is hit an ``ExecuteCommandTimeoutException``
is raised.
"""
if input is not None:
input = to_bytes(input)
p = subprocess.Popen(command, cwd=cwd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env, close_fds=kUseCloseFDs)
timerObject = None
# FIXME: Because of the way nested function scopes work in Python 2.x we
# need to use a reference to a mutable object rather than a plain
# bool. In Python 3 we could use the "nonlocal" keyword but we need
# to support Python 2 as well.
hitTimeOut = [False]
try:
if timeout > 0:
def killProcess():
# We may be invoking a shell so we need to kill the
# process and all its children.
hitTimeOut[0] = True
killProcessAndChildren(p.pid)
timerObject = threading.Timer(timeout, killProcess)
timerObject.start()
out,err = p.communicate(input=input)
exitCode = p.wait()
finally:
if timerObject != None:
timerObject.cancel()
# Ensure the resulting output is always of string type.
out = convert_string(out)
err = convert_string(err)
if hitTimeOut[0]:
raise ExecuteCommandTimeoutException(
msg='Reached timeout of {} seconds'.format(timeout),
out=out,
err=err,
exitCode=exitCode
)
# Detect Ctrl-C in subprocess.
if exitCode == -signal.SIGINT:
raise KeyboardInterrupt
return out, err, exitCode
def killProcessAndChildren(pid):
"""
This function kills a process with ``pid`` and all its
running children (recursively). It is currently implemented
using the psutil module which provides a simple platform
neutral implementation.
TODO: Reimplement this without using psutil so we can
remove our dependency on it.
"""
import psutil
try:
psutilProc = psutil.Process(pid)
# Handle the different psutil API versions
try:
# psutil >= 2.x
children_iterator = psutilProc.children(recursive=True)
except AttributeError:
# psutil 1.x
children_iterator = psutilProc.get_children(recursive=True)
for child in children_iterator:
try:
child.kill()
except psutil.NoSuchProcess:
pass
psutilProc.kill()
except psutil.NoSuchProcess:
pass
def executeCommandVerbose(cmd, *args, **kwargs):
"""
Execute a command and print its output on failure.
"""
out, err, exitCode = executeCommand(cmd, *args, **kwargs)
if exitCode != 0:
report = makeReport(cmd, out, err, exitCode)
report += "\n\nFailed!"
sys.stderr.write('%s\n' % report)
return out, err, exitCode