Initial commit of a new testsuite feature: test categories.

This feature allows us to group test cases into logical groups (categories), and to only run a subset of test cases based on these categories.

Each test-case can have a new method getCategories(self): which returns a list of strings that are the categories to which the test case belongs.
If a test-case does not provide its own categories, we will look for categories in the class that contains the test case.
If that fails too, the default implementation looks for a .category file, which contains a comma separated list of strings.
The test suite will recurse look for .categories up until the top level directory (which we guarantee will have an empty .category file).

The driver dotest.py has a new --category <foo> option, which can be repeated, and specifies which categories of tests you want to run.
(example: ./dotest.py --category objc --category expression)

All tests that do not belong to any specified category will be skipped. Other filtering options still exist and should not interfere with category filtering.
A few tests have been categorized. Feel free to categorize others, and to suggest new categories that we could want to use.

All categories need to be validly defined in dotest.py, or the test suite will refuse to run when you use them as arguments to --category.

In the end, failures will be reported on a per-category basis, as well as in the usual format.

This is the very first stage of this feature. Feel free to chime in with ideas for improvements!

llvm-svn: 164403

lldb/test/attic/tester.py

@@ -98,11 +98,15 @@ class LLDBTestCase(unittest.TestCase):
     else:
       self.fail("Command " + command + " returned an error")
       return None
+  def getCategories(self):
+    return []
 
 class SanityCheckTestCase(LLDBTestCase):
   def runTest(self):
     ret = self.runCommand("show arch", "show-arch")
     #print ret
+  def getCategories(self):
+    return []
 
 suite = unittest.TestLoader().loadTestsFromTestCase(SanityCheckTestCase)
 unittest.TextTestRunner(verbosity=2).run(suite)

lldb/test/dotest.py

@@ -66,6 +66,17 @@ class _WritelnDecorator(object):
 # Global variables:
 #
 
+# Dictionary of categories
+# When you define a new category for your testcases, be sure to add it here, or the test suite
+# will gladly complain as soon as you try to use it. This allows us to centralize which categories
+# exist, and to provide a description for each one
+validCategories = {
+'dataformatters':'Tests related to the type command and the data formatters subsystem',
+'expression':'Tests related to the expression parser',
+'objc':'Tests related to the Objective-C programming language support',
+'pyapi':'Tests related to the Python API'
+}
+
 # The test suite.
 suite = unittest2.TestSuite()
 
@@ -94,6 +105,13 @@ blacklist = None
 # The dictionary as a result of sourcing blacklistFile.
 blacklistConfig = {}
 
+# The list of categories we said we care about
+categoriesList = None
+# set to true if we are going to use categories for cherry-picking test cases
+useCategories = False
+# use this to track per-category failures
+failuresPerCategory = {}
+
 # The config file is optional.
 configFile = None
 
@@ -296,6 +314,9 @@ def parseOptionsAndInitTestdirs():
     global dont_do_dwarf_test
     global blacklist
     global blacklistConfig
+    global categoriesList
+    global validCategories
+    global useCategories
     global configFile
     global archs
     global compilers
@@ -353,6 +374,7 @@ def parseOptionsAndInitTestdirs():
     X('-l', "Don't skip long running tests")
     group.add_argument('-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite')
     group.add_argument('-X', metavar='directory', help="Exclude a directory from consideration for test discovery. -X types => if 'types' appear in the pathname components of a potential testfile, it will be ignored")
+    group.add_argument('-G', '--category', metavar='category', action='append', dest='categoriesList', help=textwrap.dedent('''Specify categories of test cases of interest. Can be specified more than once.'''))
 
     # Configuration options
     group = parser.add_argument_group('Configuration options')
@@ -400,6 +422,16 @@ def parseOptionsAndInitTestdirs():
         else:
             archs = [platform_machine]
 
+    if args.categoriesList:
+        for category in args.categoriesList:
+            if not(category in validCategories):
+                print "fatal error: category '" + category + "' is not a valid category - edit dotest.py or correct your invocation"
+                sys.exit(1)
+        categoriesList = set(args.categoriesList)
+        useCategories = True
+    else:
+        categoriesList = []
+
     if args.compilers:
         compilers = args.compilers
     else:
@@ -1228,7 +1260,34 @@ for ia in range(len(archs) if iterArchs else 1):
                 else:
                     return str(test)
 
+            def getCategoriesForTest(self,test):
+                if hasattr(test,"getCategories"):
+                    test_categories = test.getCategories()
+                elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"):
+                    test_categories = test.__self__.getCategories()
+                else:
+                    test_categories = []
+                if test_categories == None:
+                    test_categories = []
+                return test_categories
+
+            def shouldSkipBecauseOfCategories(self,test):
+                global useCategories
+                import inspect
+                if useCategories:
+                    global categoriesList
+                    test_categories = self.getCategoriesForTest(test)
+                    if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0:
+                        return True
+                return False
+
+            def hardMarkAsSkipped(self,test):
+                getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
+                getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
+
             def startTest(self, test):
+                if self.shouldSkipBecauseOfCategories(test):
+                    self.hardMarkAsSkipped(test)
                 self.counter += 1
                 if self.showAll:
                     self.stream.write(self.fmt % self.counter)
@@ -1244,11 +1303,19 @@ for ia in range(len(archs) if iterArchs else 1):
 
             def addFailure(self, test, err):
                 global sdir_has_content
+                global failuresPerCategory
                 sdir_has_content = True
                 super(LLDBTestResult, self).addFailure(test, err)
                 method = getattr(test, "markFailure", None)
                 if method:
                     method()
+                if useCategories:
+                    test_categories = self.getCategoriesForTest(test)
+                    for category in test_categories:
+                        if category in failuresPerCategory:
+                            failuresPerCategory[category] = failuresPerCategory[category] + 1
+                        else:
+                            failuresPerCategory[category] = 1
 
             def addExpectedFailure(self, test, err):
                 global sdir_has_content
@@ -1296,6 +1363,11 @@ if sdir_has_content:
     sys.stderr.write("Session logs for test failures/errors/unexpected successes"
                      " can be found in directory '%s'\n" % sdir_name)
 
+if useCategories and len(failuresPerCategory) > 0:
+    sys.stderr.write("Failures per category:\n")
+    for category in failuresPerCategory:
+        sys.stderr.write("%s - %d\n" % (category,failuresPerCategory[category]))
+
 fname = os.path.join(sdir_name, "TestFinished")
 with open(fname, "w") as f:
     print >> f, "Test finished at: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")

lldb/test/example/TestSequenceFunctions.py

@@ -29,6 +29,8 @@ class SequenceFunctionsTestCase(unittest.TestCase):
         for element in random.sample(self.seq, 5):
             self.assertTrue(element in self.seq)
 
+    def getCategories(self):
+        return []
 
 if __name__ == '__main__':
     unittest.main()

lldb/test/expression_command/.categories

@@ -0,0 +1 @@
+expression

lldb/test/functionalities/completion/TestCompletion.py

@@ -15,8 +15,11 @@ class CommandLineCompletionTestCase(TestBase):
     @classmethod
     def classCleanup(cls):
         """Cleanup the test byproducts."""
-        os.remove("child_send.txt")
-        os.remove("child_read.txt")
+        try:
+            os.remove("child_send.txt")
+            os.remove("child_read.txt")
+        except:
+            pass
 
     def test_at(self):
         """Test that 'at' completes to 'attach '."""

lldb/test/functionalities/data-formatter/.categories

@@ -0,0 +1 @@
+dataformatters

lldb/test/functionalities/data-formatter/data-formatter-objc/.categories

@@ -0,0 +1 @@
+dataformatter,objc

lldb/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjC.py

@@ -89,6 +89,8 @@ class ObjCDataFormatterTestCase(TestBase):
         """Test common cases of expression parser <--> formatters interaction."""
         self.buildDsym()
         self.expr_objc_data_formatter_commands()
+        def getCategories(self):
+            return ['dataformatters','expression','objc']
 
     @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
     @dwarf_test
@@ -96,6 +98,8 @@ class ObjCDataFormatterTestCase(TestBase):
         """Test common cases of expression parser <--> formatters interaction."""
         self.buildDwarf()
         self.expr_objc_data_formatter_commands()
+        def getCategories(self):
+            return ['dataformatters','expression','objc']
 
     def setUp(self):
         # Call super's setUp().

lldb/test/functionalities/expr-doesnt-deadlock/.categories

@@ -0,0 +1 @@
+expression

lldb/test/functionalities/single-quote-in-filename-to-lldb/TestSingleQuoteInFilename.py

@@ -16,9 +16,12 @@ class SingleQuoteInCommandLineTestCase(TestBase):
     @classmethod
     def classCleanup(cls):
         """Cleanup the test byproducts."""
-        os.remove("child_send.txt")
-        os.remove("child_read.txt")
-        os.remove(cls.myexe)
+        try:
+            os.remove("child_send.txt")
+            os.remove("child_read.txt")
+            os.remove(cls.myexe)
+        except:
+            pass
 
     def test_lldb_invocation_with_single_quote_in_filename(self):
         """Test that 'lldb my_file_name' works where my_file_name is a string with a single quote char in it."""

lldb/test/lang/objc/.categories

@@ -0,0 +1 @@
+objc

lldb/test/lldbtest.py

@@ -946,6 +946,30 @@ class TestBase(Base):
                 waitTime = float(os.environ["LLDB_TIME_WAIT_BETWEEN_TEST_CASES"])
             time.sleep(waitTime)
 
+    # Returns the list of categories to which this test case belongs
+    # by default, look for a ".categories" file, and read its contents
+    # if no such file exists, traverse the hierarchy - we guarantee
+    # a .categories to exist at the top level directory so we do not end up
+    # looping endlessly - subclasses are free to define their own categories
+    # in whatever way makes sense to them
+    def getCategories(self):
+        import inspect
+        import os.path
+        folder = inspect.getfile(self.__class__)
+        folder = os.path.dirname(folder)
+        while folder != '/':
+                categories_file_name = os.path.join(folder,".categories")
+                if os.path.exists(categories_file_name):
+                        categories_file = open(categories_file_name,'r')
+                        categories = categories_file.readline()
+                        categories_file.close()
+                        categories = str.replace(categories,'\n','')
+                        categories = str.replace(categories,'\r','')
+                        return categories.split(',')
+                else:
+                        folder = os.path.dirname(folder)
+                        continue
+
     def setUp(self):
         #import traceback
         #traceback.print_stack()

lldb/test/python_api/.categories

@@ -0,0 +1 @@
+pyapi