Parameterize the iteration count used when running benchmarks, instead of hard-coded inside the test case.

Add a '-y count' option to the test driver for this purpose.  An example:

 $  ./dotest.py -v -y 25 +b -p TestDisassembly.py

...

----------------------------------------------------------------------
Collected 2 tests

1: test_run_gdb_then_lldb (TestDisassembly.DisassembleDriverMainLoop)
   Test disassembly on a large function with lldb vs. gdb. ... 
gdb benchmark: Avg: 0.226305 (Laps: 25, Total Elapsed Time: 5.657614)
lldb benchmark: Avg: 0.113864 (Laps: 25, Total Elapsed Time: 2.846606)
lldb_avg/gdb_avg: 0.503146
ok
2: test_run_lldb_then_gdb (TestDisassembly.DisassembleDriverMainLoop)
   Test disassembly on a large function with lldb vs. gdb. ... 
lldb benchmark: Avg: 0.113008 (Laps: 25, Total Elapsed Time: 2.825201)
gdb benchmark: Avg: 0.225240 (Laps: 25, Total Elapsed Time: 5.631001)
lldb_avg/gdb_avg: 0.501723
ok

----------------------------------------------------------------------
Ran 2 tests in 41.346s

OK

llvm-svn: 142598

lldb/test/benchmarks/disassembly/TestDisassembly.py

@@ -16,14 +16,17 @@ class DisassembleDriverMainLoop(BenchBase):
         self.function = 'Driver::MainLoop()'
         self.lldb_avg = None
         self.gdb_avg = None
+        self.count = lldb.bmIterationCount
+        if self.count <= 0:
+            self.count = 5
 
     @benchmarks_test
     def test_run_lldb_then_gdb(self):
         """Test disassembly on a large function with lldb vs. gdb."""
         print
-        self.run_lldb_disassembly(self.exe, self.function, 5)
+        self.run_lldb_disassembly(self.exe, self.function, self.count)
         print "lldb benchmark:", self.stopwatch
-        self.run_gdb_disassembly(self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.exe, self.function, self.count)
         print "gdb benchmark:", self.stopwatch
         print "lldb_avg/gdb_avg: %f" % (self.lldb_avg/self.gdb_avg)
 
@@ -31,9 +34,9 @@ class DisassembleDriverMainLoop(BenchBase):
     def test_run_gdb_then_lldb(self):
         """Test disassembly on a large function with lldb vs. gdb."""
         print
-        self.run_gdb_disassembly(self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.exe, self.function, self.count)
         print "gdb benchmark:", self.stopwatch
-        self.run_lldb_disassembly(self.exe, self.function, 5)
+        self.run_lldb_disassembly(self.exe, self.function, self.count)
         print "lldb benchmark:", self.stopwatch
         print "lldb_avg/gdb_avg: %f" % (self.lldb_avg/self.gdb_avg)
 

lldb/test/benchmarks/disassembly/TestFlintVsSlateGDBDisassembly.py

@@ -18,15 +18,18 @@ class FlintVsSlateGDBDisassembly(BenchBase):
         self.function = 'Driver::MainLoop()'
         self.gdb_41_avg = None
         self.gdb_42_avg = None
+        self.count = lldb.bmIterationCount
+        if self.count <= 0:
+            self.count = 5
 
     @benchmarks_test
     def test_run_41_then_42(self):
         """Test disassembly on a large function with 4.1 vs. 4.2's gdb."""
         print
-        self.run_gdb_disassembly(self.gdb_41_exe, self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.gdb_41_exe, self.exe, self.function, self.count)
         print "4.1 gdb benchmark:", self.stopwatch
         self.gdb_41_avg = self.stopwatch.avg()
-        self.run_gdb_disassembly(self.gdb_42_exe, self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.gdb_42_exe, self.exe, self.function, self.count)
         print "4.2 gdb benchmark:", self.stopwatch
         self.gdb_42_avg = self.stopwatch.avg()
         print "gdb_42_avg/gdb_41_avg: %f" % (self.gdb_42_avg/self.gdb_41_avg)
@@ -35,10 +38,10 @@ class FlintVsSlateGDBDisassembly(BenchBase):
     def test_run_42_then_41(self):
         """Test disassembly on a large function with 4.1 vs. 4.2's gdb."""
         print
-        self.run_gdb_disassembly(self.gdb_42_exe, self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.gdb_42_exe, self.exe, self.function, self.count)
         print "4.2 gdb benchmark:", self.stopwatch
         self.gdb_42_avg = self.stopwatch.avg()
-        self.run_gdb_disassembly(self.gdb_41_exe, self.exe, self.function, 5)
+        self.run_gdb_disassembly(self.gdb_41_exe, self.exe, self.function, self.count)
         print "4.1 gdb benchmark:", self.stopwatch
         self.gdb_41_avg = self.stopwatch.avg()
         print "gdb_42_avg/gdb_41_avg: %f" % (self.gdb_42_avg/self.gdb_41_avg)

lldb/test/benchmarks/example/TestRepeatedExprs.py

@@ -16,6 +16,9 @@ class RepeatedExprsCase(BenchBase):
         self.line_to_break = line_number(self.source, '// Set breakpoint here.')
         self.lldb_avg = None
         self.gdb_avg = None
+        self.count = lldb.bmIterationCount
+        if self.count <= 0:
+            self.count = 100
 
     @benchmarks_test
     def test_compare_lldb_to_gdb(self):
@@ -24,9 +27,9 @@ class RepeatedExprsCase(BenchBase):
         self.exe_name = 'a.out'
 
         print
-        self.run_lldb_repeated_exprs(self.exe_name, 100)
+        self.run_lldb_repeated_exprs(self.exe_name, self.count)
         print "lldb benchmark:", self.stopwatch
-        self.run_gdb_repeated_exprs(self.exe_name, 100)
+        self.run_gdb_repeated_exprs(self.exe_name, self.count)
         print "gdb benchmark:", self.stopwatch
         print "lldb_avg/gdb_avg: %f" % (self.lldb_avg/self.gdb_avg)
 

lldb/test/benchmarks/stepping/TestRunHooksThenSteppings.py

@@ -12,12 +12,15 @@ class RunHooksThenSteppingsBench(BenchBase):
 
     def setUp(self):
         BenchBase.setUp(self)
+        self.count = lldb.bmIterationCount
+        if self.count <= 0:
+            self.count = 50
 
     @benchmarks_test
     def test_lldb_runhooks_then_steppings(self):
         """Test lldb steppings on a large executable."""
         print
-        self.run_lldb_runhooks_then_steppings(50)
+        self.run_lldb_runhooks_then_steppings(self.count)
         print "lldb stepping benchmark:", self.stopwatch
 
     def run_lldb_runhooks_then_steppings(self, count):

lldb/test/benchmarks/stepping/TestSteppingSpeed.py

@@ -26,6 +26,10 @@ class SteppingSpeedBench(BenchBase):
             else:
                 self.break_spec = '-n main'
 
+        self.count = lldb.bmIterationCount
+        if self.count <= 0:
+            self.count = 50
+
         #print "self.exe=%s" % self.exe
         #print "self.break_spec=%s" % self.break_spec
 
@@ -33,7 +37,7 @@ class SteppingSpeedBench(BenchBase):
     def test_run_lldb_steppings(self):
         """Test lldb steppings on a large executable."""
         print
-        self.run_lldb_steppings(self.exe, self.break_spec, 50)
+        self.run_lldb_steppings(self.exe, self.break_spec, self.count)
         print "lldb stepping benchmark:", self.stopwatch
 
     def run_lldb_steppings(self, exe, break_spec, count):

lldb/test/dotest.py

@@ -107,6 +107,8 @@ dumpSysPath = False
 bmExecutable = None
 # The breakpoint specification of bmExecutable, as specified by the '-x' option.
 bmBreakpointSpec = None
+# The benchamrk iteration count, as specified by the '-y' option.
+bmIterationCount = -1
 
 # By default, failfast is False.  Use '-F' to overwrite it.
 failfast = False
@@ -210,6 +212,9 @@ where options:
 -v   : do verbose mode of unittest framework (print out each test case invocation)
 -x   : specify the breakpoint specification for the benchmark executable;
        see also '-e', which provides the full path of the executable
+-y   : specify the iteration count used to collect our benchmarks; an example is
+       the number of times to do 'thread step-over' to measure stepping speed
+       see also '-e' and '-x' options
 -w   : insert some wait time (currently 0.5 sec) between consecutive test cases
 -#   : Repeat the test suite for a specified number of times
 
@@ -321,6 +326,7 @@ def parseOptionsAndInitTestdirs():
     global dumpSysPath
     global bmExecutable
     global bmBreakpointSpec
+    global bmIterationCount
     global failfast
     global filters
     global fs4all
@@ -478,6 +484,13 @@ def parseOptionsAndInitTestdirs():
                 usage()
             bmBreakpointSpec = sys.argv[index]
             index += 1
+        elif sys.argv[index].startswith('-y'):
+            # Increment by 1 to fetch the the benchmark iteration count.
+            index += 1
+            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
+                usage()
+            bmIterationCount = int(sys.argv[index])
+            index += 1
         elif sys.argv[index].startswith('-#'):
             # Increment by 1 to fetch the repeat count argument.
             index += 1
@@ -902,9 +915,10 @@ lldb.dont_do_python_api_test = dont_do_python_api_test
 lldb.just_do_python_api_test = just_do_python_api_test
 lldb.just_do_benchmarks_test = just_do_benchmarks_test
 
-# Put bmExecutable and bmBreakpointSpec into the lldb namespace, too.
+# Put bmExecutable, bmBreakpointSpec, and bmIterationCount into the lldb namespace, too.
 lldb.bmExecutable = bmExecutable
 lldb.bmBreakpointSpec = bmBreakpointSpec
+lldb.bmIterationCount = bmIterationCount
 
 # And don't forget the runHooks!
 lldb.runHooks = runHooks