forked from OSchip/llvm-project
Roll dosep.py parallel test runner into dotest.py command line
See the following for details: http://reviews.llvm.org/D12587 llvm-svn: 246794
This commit is contained in:
Todd Fiala
parent
222edc66d6
commit
fed95660f3
|
|
@ -62,16 +62,16 @@ endif()
|
|||
|
||||
add_python_test_target(check-lldb-single
|
||||
${LLDB_SOURCE_DIR}/test/dotest.py
|
||||
"${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
|
||||
"--no-multiprocess;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
|
||||
"Testing LLDB with args: ${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
|
||||
)
|
||||
|
||||
set(LLDB_DOSEP_ARGS -o;\"-q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}\")
|
||||
set(LLDB_DOTEST_ARGS -q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS})
|
||||
|
||||
# If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable
|
||||
# output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target.
|
||||
add_python_test_target(check-lldb
|
||||
${LLDB_SOURCE_DIR}/test/dosep.py
|
||||
"${LLDB_DOSEP_ARGS}"
|
||||
"Testing LLDB (with a separate subprocess per test)"
|
||||
${LLDB_SOURCE_DIR}/test/dotest.py
|
||||
"${LLDB_DOTEST_ARGS}"
|
||||
"Testing LLDB (parallel execution, with a separate subprocess per test)"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -30,4 +30,4 @@ clean::
|
|||
#----------------------------------------------------------------------
|
||||
check-local::
|
||||
rm -rf lldb-test-traces
|
||||
python $(PROJ_SRC_DIR)/dosep.py -o "--executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))"
|
||||
python $(PROJ_SRC_DIR)/dotest.py --executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ import fnmatch
|
|||
import platform
|
||||
import re
|
||||
import dotest_args
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
|
@ -131,8 +130,6 @@ def parse_test_results(output):
|
|||
result, re.MULTILINE)
|
||||
unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes",
|
||||
result, re.MULTILINE)
|
||||
this_fail_count = 0
|
||||
this_error_count = 0
|
||||
if pass_count is not None:
|
||||
passes = passes + int(pass_count.group(1))
|
||||
if fail_count is not None:
|
||||
|
|
@ -183,7 +180,7 @@ def process_dir(root, files, test_root, dotest_argv):
|
|||
script_file = os.path.join(test_root, "dotest.py")
|
||||
command = ([sys.executable, script_file] +
|
||||
dotest_argv +
|
||||
["-p", name, root])
|
||||
["--inferior", "-p", name, root])
|
||||
|
||||
timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
|
||||
|
||||
|
|
@ -201,7 +198,7 @@ def process_dir(root, files, test_root, dotest_argv):
|
|||
if status != ePassed]
|
||||
unexpected_passes = [name for name, _, _, _, unexpected_successes in results
|
||||
if unexpected_successes > 0]
|
||||
|
||||
|
||||
pass_count = sum([result[2] for result in results])
|
||||
fail_count = sum([result[3] for result in results])
|
||||
|
||||
|
|
@ -284,7 +281,6 @@ def getExpectedTimeouts(platform_name):
|
|||
else:
|
||||
m = re.search('remote-(\w+)', platform_name)
|
||||
target = m.group(1)
|
||||
remote = True
|
||||
|
||||
expected_timeout = set()
|
||||
|
||||
|
|
@ -358,7 +354,27 @@ def find(pattern, path):
|
|||
return result
|
||||
|
||||
|
||||
def main():
|
||||
def main(print_details_on_success, num_threads, test_subdir):
|
||||
"""Run dotest.py in inferior mode in parallel.
|
||||
|
||||
@param print_details_on_success the parsed value of the output-on-success
|
||||
command line argument. When True, details of a successful dotest inferior
|
||||
are printed even when everything succeeds. The normal behavior is to
|
||||
not print any details when all the inferior tests pass.
|
||||
|
||||
@param num_threads the parsed value of the num-threads command line
|
||||
argument.
|
||||
|
||||
@param test_subdir optionally specifies a subdir to limit testing
|
||||
within. May be None if the entire test tree is to be used. This subdir
|
||||
is assumed to be relative to the lldb/test root of the test hierarchy.
|
||||
"""
|
||||
|
||||
dotest_argv = sys.argv[1:]
|
||||
|
||||
global output_on_success
|
||||
output_on_success = print_details_on_success
|
||||
|
||||
# We can't use sys.path[0] to determine the script directory
|
||||
# because it doesn't work under a debugger
|
||||
test_directory = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -382,37 +398,8 @@ Run lldb test suite using a separate process for each test file.
|
|||
E.g., export LLDB_TEST_TIMEOUT=0
|
||||
or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
|
||||
""")
|
||||
parser.add_option(
|
||||
'-o', '--options',
|
||||
type='string', action='store',
|
||||
dest='dotest_options',
|
||||
help="""The options passed to 'dotest.py' if specified.""")
|
||||
|
||||
parser.add_option(
|
||||
'-s', '--output-on-success',
|
||||
action='store_true',
|
||||
dest='output_on_success',
|
||||
default=False,
|
||||
help="""Print full output of 'dotest.py' even when it succeeds.""")
|
||||
|
||||
parser.add_option(
|
||||
'-t', '--threads',
|
||||
type='int',
|
||||
dest='num_threads',
|
||||
help="""The number of threads to use when running tests separately.""")
|
||||
|
||||
opts, args = parser.parse_args()
|
||||
dotest_option_string = opts.dotest_options
|
||||
|
||||
is_posix = (os.name == "posix")
|
||||
dotest_argv = (shlex.split(dotest_option_string, posix=is_posix)
|
||||
if dotest_option_string
|
||||
else [])
|
||||
|
||||
parser = dotest_args.create_parser()
|
||||
global dotest_options
|
||||
global output_on_success
|
||||
output_on_success = opts.output_on_success
|
||||
dotest_options = dotest_args.parse_args(parser, dotest_argv)
|
||||
|
||||
if not dotest_options.s:
|
||||
|
|
@ -428,19 +415,17 @@ Run lldb test suite using a separate process for each test file.
|
|||
session_dir = os.path.join(os.getcwd(), dotest_options.s)
|
||||
|
||||
# The root directory was specified on the command line
|
||||
if len(args) == 0:
|
||||
test_subdir = test_directory
|
||||
if test_subdir and len(test_subdir) > 0:
|
||||
test_subdir = os.path.join(test_directory, test_subdir)
|
||||
else:
|
||||
test_subdir = os.path.join(test_directory, args[0])
|
||||
test_subdir = test_directory
|
||||
|
||||
# clean core files in test tree from previous runs (Linux)
|
||||
cores = find('core.*', test_subdir)
|
||||
for core in cores:
|
||||
os.unlink(core)
|
||||
|
||||
if opts.num_threads:
|
||||
num_threads = opts.num_threads
|
||||
else:
|
||||
if not num_threads:
|
||||
num_threads_str = os.environ.get("LLDB_TEST_THREADS")
|
||||
if num_threads_str:
|
||||
num_threads = int(num_threads_str)
|
||||
|
|
@ -511,4 +496,8 @@ Run lldb test suite using a separate process for each test file.
|
|||
sys.exit(exit_code)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
sys.stderr.write(
|
||||
"error: dosep.py no longer supports being called directly. "
|
||||
"Please call dotest.py directly. The dosep.py-specific arguments "
|
||||
"have been added under the Parallel processing arguments.\n")
|
||||
sys.exit(128)
|
||||
|
|
|
|||
|
|
@ -29,8 +29,6 @@ import progress
|
|||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
import inspect
|
||||
import unittest2
|
||||
import lldbtest_config
|
||||
|
|
@ -245,6 +243,13 @@ lldb_platform_name = None
|
|||
lldb_platform_url = None
|
||||
lldb_platform_working_dir = None
|
||||
|
||||
# Parallel execution settings
|
||||
is_inferior_test_runner = False
|
||||
multiprocess_test_subdir = None
|
||||
num_threads = None
|
||||
output_on_success = False
|
||||
no_multiprocess_test_runner = False
|
||||
|
||||
def usage(parser):
|
||||
parser.print_help()
|
||||
if verbose > 0:
|
||||
|
|
@ -485,6 +490,11 @@ def parseOptionsAndInitTestdirs():
|
|||
global lldb_platform_url
|
||||
global lldb_platform_working_dir
|
||||
global setCrashInfoHook
|
||||
global is_inferior_test_runner
|
||||
global multiprocess_test_subdir
|
||||
global num_threads
|
||||
global output_on_success
|
||||
global no_multiprocess_test_runner
|
||||
|
||||
do_help = False
|
||||
|
||||
|
|
@ -493,7 +503,7 @@ def parseOptionsAndInitTestdirs():
|
|||
|
||||
parser = dotest_args.create_parser()
|
||||
args = dotest_args.parse_args(parser, sys.argv[1:])
|
||||
|
||||
|
||||
if args.unset_env_varnames:
|
||||
for env_var in args.unset_env_varnames:
|
||||
if env_var in os.environ:
|
||||
|
|
@ -606,7 +616,7 @@ def parseOptionsAndInitTestdirs():
|
|||
|
||||
if args.d:
|
||||
sys.stdout.write("Suspending the process %d to wait for debugger to attach...\n" % os.getpid())
|
||||
sys.stdout.flush()
|
||||
sys.stdout.flush()
|
||||
os.kill(os.getpid(), signal.SIGSTOP)
|
||||
|
||||
if args.e:
|
||||
|
|
@ -740,6 +750,21 @@ def parseOptionsAndInitTestdirs():
|
|||
if dont_do_lldbmi_test and just_do_lldbmi_test:
|
||||
usage(parser)
|
||||
|
||||
if args.no_multiprocess:
|
||||
no_multiprocess_test_runner = True
|
||||
|
||||
if args.inferior:
|
||||
is_inferior_test_runner = True
|
||||
|
||||
if args.output_on_success:
|
||||
output_on_success = True
|
||||
|
||||
if args.num_threads:
|
||||
num_threads = args.num_threads
|
||||
|
||||
if args.test_subdir:
|
||||
multiprocess_test_subdir = args.test_subdir
|
||||
|
||||
if args.lldb_platform_name:
|
||||
lldb_platform_name = args.lldb_platform_name
|
||||
if args.lldb_platform_url:
|
||||
|
|
@ -1228,6 +1253,14 @@ def exitTestSuite(exitCode = None):
|
|||
if exitCode:
|
||||
sys.exit(exitCode)
|
||||
|
||||
|
||||
def isMultiprocessTestRunner():
|
||||
# We're not multiprocess when we're either explicitly
|
||||
# the inferior (as specified by the multiprocess test
|
||||
# runner) OR we've been told to skip using the multiprocess
|
||||
# test runner
|
||||
return not (is_inferior_test_runner or no_multiprocess_test_runner)
|
||||
|
||||
# On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
|
||||
# does not exist before proceeding to running the test suite.
|
||||
if sys.platform.startswith("darwin"):
|
||||
|
|
@ -1239,6 +1272,14 @@ if sys.platform.startswith("darwin"):
|
|||
# then, we walk the directory trees and collect the tests into our test suite.
|
||||
#
|
||||
parseOptionsAndInitTestdirs()
|
||||
|
||||
# If we are running as the multiprocess test runner, kick off the
|
||||
# multiprocess test runner here.
|
||||
if isMultiprocessTestRunner():
|
||||
import dosep
|
||||
dosep.main(output_on_success, num_threads, multiprocess_test_subdir)
|
||||
raise "should never get here"
|
||||
|
||||
setupSysPath()
|
||||
setupCrashInfoHook()
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,33 @@ def create_parser():
|
|||
group.set_defaults(disable_crash_dialog=True)
|
||||
group.set_defaults(hide_inferior_console=True)
|
||||
|
||||
group = parser.add_argument_group('Parallel execution options')
|
||||
group.add_argument(
|
||||
'--inferior',
|
||||
action='store_true',
|
||||
help=('specify this invocation is a multiprocess inferior, '
|
||||
'used internally'))
|
||||
group.add_argument(
|
||||
'--no-multiprocess',
|
||||
action='store_true',
|
||||
help='skip running the multiprocess test runner')
|
||||
group.add_argument(
|
||||
'--output-on-success',
|
||||
action='store_true',
|
||||
help=('print full output of the dotest.py inferior, '
|
||||
'even when all tests succeed'))
|
||||
group.add_argument(
|
||||
'--threads',
|
||||
type=int,
|
||||
dest='num_threads',
|
||||
help=('The number of threads/processes to use when running tests '
|
||||
'separately, defaults to the number of CPU cores available'))
|
||||
parser.add_argument(
|
||||
'--test-subdir',
|
||||
action='store',
|
||||
help='Specify a test subdirectory to use relative to the test root dir'
|
||||
)
|
||||
|
||||
# Remove the reference to our helper function
|
||||
del X
|
||||
|
||||
|
|
|
|||
|
|
@ -67,19 +67,30 @@
|
|||
</code>
|
||||
|
||||
<p>
|
||||
Besides <code>dotest.py</code>, there is also <code>dosep.py</code>, which runs
|
||||
multiple instances of <code>dotest.py</code> in parallel, thereby greatly
|
||||
decreasing the time it takes to run the full testsuite. The number of concurrent
|
||||
tests is controlled by the <code>LLDB_TEST_THREADS</code> environment variable or
|
||||
the <code>--threads</code> command line parameter. The default value is the number
|
||||
of CPUs on your system. To pass additional options to <code>dotest.py</code>,
|
||||
specify those options as an <code>-o</code> argument to <code>dosep.py</code>. For
|
||||
example, the command
|
||||
The dotest.py script runs tests in parallel by default.
|
||||
To disable the parallel test running feature, use the
|
||||
<code>--no-multiprocess</code> flag. The number of
|
||||
concurrent tests is controlled by
|
||||
the <code>LLDB_TEST_THREADS</code> environment variable
|
||||
or the <code>--threads</code> command line parameter.
|
||||
The default value is the number of CPU cores on your
|
||||
system.
|
||||
</p>
|
||||
<code>python dosep.py -o "--executable bin/lldb -C bin/clang"</code>
|
||||
<p>
|
||||
will specify the lldb and clang executables to test for each dotest invocation.
|
||||
<code>ninja check-lldb</code> is wrapper around <code>dosep.py</code>.
|
||||
The parallel test running feature will handle an
|
||||
additional <code>--test-subdir SUBDIR</code> arg. When
|
||||
specified, SUBDIR is relative to the root test directory
|
||||
and will limit all parallel test running to that
|
||||
sudirectory's tree of tests.
|
||||
</p>
|
||||
<p>
|
||||
The parallel test runner will run all tests within a
|
||||
given directory serially, but will run multiple
|
||||
directories concurrently. Thus, as a test writer, we
|
||||
provide serialized test run semantics within a
|
||||
directory. Note child directories are considered
|
||||
entirely separate, so two child directories could be
|
||||
running in parallel with a parent directory.
|
||||
</p>
|
||||
|
||||
<h3>Running the test-suite remotely</h3>
|
||||
|
|
|
|||
Loading…
Reference in New Issue