Roll dosep.py parallel test runner into dotest.py command line

See the following for details:
http://reviews.llvm.org/D12587

llvm-svn: 246794

lldb/test/CMakeLists.txt

@@ -62,16 +62,16 @@ endif()
 
 add_python_test_target(check-lldb-single
   ${LLDB_SOURCE_DIR}/test/dotest.py
-  "${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
+  "--no-multiprocess;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
   "Testing LLDB with args: ${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
   )
 
-set(LLDB_DOSEP_ARGS -o;\"-q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}\")
+set(LLDB_DOTEST_ARGS -q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS})
 
 # If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable
 # output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target.
 add_python_test_target(check-lldb
-  ${LLDB_SOURCE_DIR}/test/dosep.py
+  ${LLDB_SOURCE_DIR}/test/dotest.py
-  "${LLDB_DOSEP_ARGS}"
+  "${LLDB_DOTEST_ARGS}"
-  "Testing LLDB (with a separate subprocess per test)"
+  "Testing LLDB (parallel execution, with a separate subprocess per test)"
   )

lldb/test/Makefile

@@ -30,4 +30,4 @@ clean::
 #----------------------------------------------------------------------
 check-local::
 	rm -rf lldb-test-traces
-	python $(PROJ_SRC_DIR)/dosep.py -o "--executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))"
+	python $(PROJ_SRC_DIR)/dotest.py --executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))

lldb/test/dosep.py

@@ -38,7 +38,6 @@ import fnmatch
 import platform
 import re
 import dotest_args
-import shlex
 import subprocess
 import sys
 
@@ -131,8 +130,6 @@ def parse_test_results(output):
                                 result, re.MULTILINE)
         unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes",
                                              result, re.MULTILINE)
-        this_fail_count = 0
-        this_error_count = 0
         if pass_count is not None:
             passes = passes + int(pass_count.group(1))
         if fail_count is not None:
@@ -183,7 +180,7 @@ def process_dir(root, files, test_root, dotest_argv):
         script_file = os.path.join(test_root, "dotest.py")
         command = ([sys.executable, script_file] +
                    dotest_argv +
-                   ["-p", name, root])
+                   ["--inferior", "-p", name, root])
 
         timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
 
@@ -201,7 +198,7 @@ def process_dir(root, files, test_root, dotest_argv):
               if status != ePassed]
     unexpected_passes = [name for name, _, _, _, unexpected_successes in results
                          if unexpected_successes > 0]
-    
+
     pass_count = sum([result[2] for result in results])
     fail_count = sum([result[3] for result in results])
 
@@ -284,7 +281,6 @@ def getExpectedTimeouts(platform_name):
     else:
         m = re.search('remote-(\w+)', platform_name)
         target = m.group(1)
-        remote = True
 
     expected_timeout = set()
 
@@ -358,7 +354,27 @@ def find(pattern, path):
     return result
 
 
-def main():
+def main(print_details_on_success, num_threads, test_subdir):
+    """Run dotest.py in inferior mode in parallel.
+
+    @param print_details_on_success the parsed value of the output-on-success
+    command line argument.  When True, details of a successful dotest inferior
+    are printed even when everything succeeds.  The normal behavior is to
+    not print any details when all the inferior tests pass.
+
+    @param num_threads the parsed value of the num-threads command line
+    argument.
+
+    @param test_subdir optionally specifies a subdir to limit testing
+    within.  May be None if the entire test tree is to be used.  This subdir
+    is assumed to be relative to the lldb/test root of the test hierarchy.
+    """
+
+    dotest_argv = sys.argv[1:]
+
+    global output_on_success
+    output_on_success = print_details_on_success
+
     # We can't use sys.path[0] to determine the script directory
     # because it doesn't work under a debugger
     test_directory = os.path.dirname(os.path.realpath(__file__))
@@ -382,37 +398,8 @@ Run lldb test suite using a separate process for each test file.
        E.g., export LLDB_TEST_TIMEOUT=0
        or    export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
 """)
-    parser.add_option(
-        '-o', '--options',
-        type='string', action='store',
-        dest='dotest_options',
-        help="""The options passed to 'dotest.py' if specified.""")
-
-    parser.add_option(
-        '-s', '--output-on-success',
-        action='store_true',
-        dest='output_on_success',
-        default=False,
-        help="""Print full output of 'dotest.py' even when it succeeds.""")
-
-    parser.add_option(
-        '-t', '--threads',
-        type='int',
-        dest='num_threads',
-        help="""The number of threads to use when running tests separately.""")
-
-    opts, args = parser.parse_args()
-    dotest_option_string = opts.dotest_options
-
-    is_posix = (os.name == "posix")
-    dotest_argv = (shlex.split(dotest_option_string, posix=is_posix)
-                   if dotest_option_string
-                   else [])
-
     parser = dotest_args.create_parser()
     global dotest_options
-    global output_on_success
-    output_on_success = opts.output_on_success
     dotest_options = dotest_args.parse_args(parser, dotest_argv)
 
     if not dotest_options.s:
@@ -428,19 +415,17 @@ Run lldb test suite using a separate process for each test file.
     session_dir = os.path.join(os.getcwd(), dotest_options.s)
 
     # The root directory was specified on the command line
-    if len(args) == 0:
+    if test_subdir and len(test_subdir) > 0:
-        test_subdir = test_directory
+        test_subdir = os.path.join(test_directory, test_subdir)
     else:
-        test_subdir = os.path.join(test_directory, args[0])
+        test_subdir = test_directory
 
     # clean core files in test tree from previous runs (Linux)
     cores = find('core.*', test_subdir)
     for core in cores:
         os.unlink(core)
 
-    if opts.num_threads:
+    if not num_threads:
-        num_threads = opts.num_threads
-    else:
         num_threads_str = os.environ.get("LLDB_TEST_THREADS")
         if num_threads_str:
             num_threads = int(num_threads_str)
@@ -511,4 +496,8 @@ Run lldb test suite using a separate process for each test file.
     sys.exit(exit_code)
 
 if __name__ == '__main__':
-    main()
+    sys.stderr.write(
+        "error: dosep.py no longer supports being called directly. "
+        "Please call dotest.py directly.  The dosep.py-specific arguments "
+        "have been added under the Parallel processing arguments.\n")
+    sys.exit(128)

lldb/test/dotest.py

@@ -29,8 +29,6 @@ import progress
 import signal
 import subprocess
 import sys
-import textwrap
-import time
 import inspect
 import unittest2
 import lldbtest_config
@@ -245,6 +243,13 @@ lldb_platform_name = None
 lldb_platform_url = None
 lldb_platform_working_dir = None
 
+# Parallel execution settings
+is_inferior_test_runner = False
+multiprocess_test_subdir = None
+num_threads = None
+output_on_success = False
+no_multiprocess_test_runner = False
+
 def usage(parser):
     parser.print_help()
     if verbose > 0:
@@ -485,6 +490,11 @@ def parseOptionsAndInitTestdirs():
     global lldb_platform_url
     global lldb_platform_working_dir
     global setCrashInfoHook
+    global is_inferior_test_runner
+    global multiprocess_test_subdir
+    global num_threads
+    global output_on_success
+    global no_multiprocess_test_runner
 
     do_help = False
 
@@ -493,7 +503,7 @@ def parseOptionsAndInitTestdirs():
 
     parser = dotest_args.create_parser()
     args = dotest_args.parse_args(parser, sys.argv[1:])
-    
+
     if args.unset_env_varnames:
         for env_var in args.unset_env_varnames:
             if env_var in os.environ:
@@ -606,7 +616,7 @@ def parseOptionsAndInitTestdirs():
 
     if args.d:
         sys.stdout.write("Suspending the process %d to wait for debugger to attach...\n" % os.getpid())
-        sys.stdout.flush()    
+        sys.stdout.flush()
         os.kill(os.getpid(), signal.SIGSTOP)
 
     if args.e:
@@ -740,6 +750,21 @@ def parseOptionsAndInitTestdirs():
     if dont_do_lldbmi_test and just_do_lldbmi_test:
         usage(parser)
 
+    if args.no_multiprocess:
+        no_multiprocess_test_runner = True
+
+    if args.inferior:
+        is_inferior_test_runner = True
+
+    if args.output_on_success:
+        output_on_success = True
+
+    if args.num_threads:
+        num_threads = args.num_threads
+
+    if args.test_subdir:
+        multiprocess_test_subdir = args.test_subdir
+
     if args.lldb_platform_name:
         lldb_platform_name = args.lldb_platform_name
     if args.lldb_platform_url:
@@ -1228,6 +1253,14 @@ def exitTestSuite(exitCode = None):
     if exitCode:
         sys.exit(exitCode)
 
+
+def isMultiprocessTestRunner():
+    # We're not multiprocess when we're either explicitly
+    # the inferior (as specified by the multiprocess test
+    # runner) OR we've been told to skip using the multiprocess
+    # test runner
+    return not (is_inferior_test_runner or no_multiprocess_test_runner)
+
 # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
 # does not exist before proceeding to running the test suite.
 if sys.platform.startswith("darwin"):
@@ -1239,6 +1272,14 @@ if sys.platform.startswith("darwin"):
 # then, we walk the directory trees and collect the tests into our test suite.
 #
 parseOptionsAndInitTestdirs()
+
+# If we are running as the multiprocess test runner, kick off the
+# multiprocess test runner here.
+if isMultiprocessTestRunner():
+    import dosep
+    dosep.main(output_on_success, num_threads, multiprocess_test_subdir)
+    raise "should never get here"
+
 setupSysPath()
 setupCrashInfoHook()
 

lldb/test/dotest_args.py

@@ -107,6 +107,33 @@ def create_parser():
     group.set_defaults(disable_crash_dialog=True)
     group.set_defaults(hide_inferior_console=True)
 
+    group = parser.add_argument_group('Parallel execution options')
+    group.add_argument(
+        '--inferior',
+        action='store_true',
+        help=('specify this invocation is a multiprocess inferior, '
+              'used internally'))
+    group.add_argument(
+        '--no-multiprocess',
+        action='store_true',
+        help='skip running the multiprocess test runner')
+    group.add_argument(
+        '--output-on-success',
+        action='store_true',
+        help=('print full output of the dotest.py inferior, '
+              'even when all tests succeed'))
+    group.add_argument(
+        '--threads',
+        type=int,
+        dest='num_threads',
+        help=('The number of threads/processes to use when running tests '
+              'separately, defaults to the number of CPU cores available'))
+    parser.add_argument(
+        '--test-subdir',
+        action='store',
+        help='Specify a test subdirectory to use relative to the test root dir'
+    )
+
     # Remove the reference to our helper function
     del X
 

lldb/www/test.html

@@ -67,19 +67,30 @@
             </code>
 
             <p>
-              Besides <code>dotest.py</code>, there is also <code>dosep.py</code>, which runs
+              The dotest.py script runs tests in parallel by default.
-              multiple instances of <code>dotest.py</code> in parallel, thereby greatly
+              To disable the parallel test running feature, use the
-              decreasing the time it takes to run the full testsuite. The number of concurrent
+              <code>--no-multiprocess</code> flag. The number of
-              tests is controlled by the <code>LLDB_TEST_THREADS</code> environment variable or
+              concurrent tests is controlled by
-              the <code>--threads</code> command line parameter. The default value is the number
+              the <code>LLDB_TEST_THREADS</code> environment variable
-              of CPUs on your system. To pass additional options to <code>dotest.py</code>,
+              or the <code>--threads</code> command line parameter.
-              specify those options as an <code>-o</code> argument to <code>dosep.py</code>. For
+              The default value is the number of CPU cores on your
-              example, the command
+              system.
             </p>
-            <code>python dosep.py -o "--executable bin/lldb -C bin/clang"</code>
             <p>
-              will specify the lldb and clang executables to test for each dotest invocation.
+              The parallel test running feature will handle an
-              <code>ninja check-lldb</code> is wrapper around <code>dosep.py</code>.
+              additional <code>--test-subdir SUBDIR</code> arg.  When
+              specified, SUBDIR is relative to the root test directory
+              and will limit all parallel test running to that
+              sudirectory's tree of tests.
+            </p>
+            <p>
+              The parallel test runner will run all tests within a
+              given directory serially, but will run multiple
+              directories concurrently.  Thus, as a test writer, we
+              provide serialized test run semantics within a
+              directory.  Note child directories are considered
+              entirely separate, so two child directories could be
+              running in parallel with a parent directory.
             </p>
 
             <h3>Running the test-suite remotely</h3>