basic functionality working

contrib/TestHarness2/test_harness/app.py

@@ -1,5 +1,4 @@
 import argparse
-import random
 import sys
 import traceback
 
@@ -10,7 +9,6 @@ from test_harness.summarize import SummaryTree
 
 if __name__ == '__main__':
     try:
-        # seed the random number generator
         parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
         config.build_arguments(parser)
         # initialize arguments
@@ -21,7 +19,6 @@ if __name__ == '__main__':
                             required=False)
         args = parser.parse_args()
         config.extract_args(args)
-        random.seed(config.joshua_seed)
         test_runner = TestRunner()
         if not test_runner.run(args.stats):
             exit(1)

contrib/TestHarness2/test_harness/config.py

@@ -70,6 +70,7 @@ class ConfigValue:
 
 class Config:
     def __init__(self):
+        self.random = random.Random()
         self.kill_seconds: int = 30 * 60
         self.kill_seconds_args = {'help': 'Timeout for individual test'}
         self.buggify_on_ratio: float = 0.8
@@ -102,6 +103,7 @@ class Config:
         self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'}
         self.run_dir: Path = Path('tmp')
         self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1))))
+        self.random.seed(self.joshua_seed, version=2)
         self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'}
         self.print_coverage = False
         self.print_coverage_args = {'action': 'store_true'}
@@ -110,9 +112,16 @@ class Config:
         self.output_format: str = 'xml'
         self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']}
         self.include_test_files: str = r'.*'
+        self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'}
         self.exclude_test_files: str = r'.^'
+        self.exclude_test_files_args = {'help': 'Don\'t consider test files whose path match against the given regex'}
         self.include_test_names: str = r'.*'
+        self.include_test_names_args = {'help': 'Only consider tests whose names match against the given regex'}
         self.exclude_test_names: str = r'.^'
+        self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'}
+        self.max_stderr_bytes: int = 1000
+        self.write_stats: bool = True
+        self.read_stats: bool = True
         self.config_map = self._build_map()
 
     def _build_map(self):
@@ -151,6 +160,8 @@ class Config:
         for val in self.config_map.values():
             k, v = val.get_value(args)
             config.__setattr__(k, v)
+            if k == 'joshua_seed':
+                self.random.seed(self.joshua_seed, version=2)
 
 
 config = Config()

contrib/TestHarness2/test_harness/run.py

@@ -8,21 +8,22 @@ import os
 import resource
 import shutil
 import subprocess
-import random
 import re
 import sys
 import threading
 import time
 import uuid
 
-from test_harness import version
+from functools import total_ordering
-from test_harness.config import config
-from typing import List, Pattern, Callable, OrderedDict, Dict
 from pathlib import Path
+from test_harness.version import Version
+from test_harness.config import config
+from typing import List, Pattern, OrderedDict, Dict
 
 from test_harness.summarize import Summary, SummaryTree
 
 
+@total_ordering
 class TestDescription:
     def __init__(self, path: Path, name: str, priority: float):
         self.paths: List[Path] = [path]
@@ -31,6 +32,18 @@ class TestDescription:
         # we only measure in seconds. Otherwise, keeping determinism will be difficult
         self.total_runtime: int = 0
 
+    def __lt__(self, other):
+        if isinstance(other, TestDescription):
+            return self.name < other.name
+        else:
+            return self.name < str(other)
+
+    def __eq__(self, other):
+        if isinstance(other, TestDescription):
+            return self.name < other.name
+        else:
+            return self.name < str(other.name)
+
 
 class StatFetcher:
     def __init__(self, tests: OrderedDict[str, TestDescription]):
@@ -68,11 +81,8 @@ class FileStatsFetcher(StatFetcher):
             json.dump(self.last_state, f)
 
 
-StatFetcherCreator = Callable[[OrderedDict[str, TestDescription]], StatFetcher]
-
-
 class TestPicker:
-    def __init__(self, test_dir: Path, fetcher: StatFetcherCreator):
+    def __init__(self, test_dir: Path):
         if not test_dir.exists():
             raise RuntimeError('{} is neither a directory nor a file'.format(test_dir))
         self.include_files_regex = re.compile(config.include_test_files)
@@ -87,7 +97,8 @@ class TestPicker:
         for subdir in self.test_dir.iterdir():
             if subdir.is_dir() and subdir.name in config.test_dirs:
                 self.walk_test_dir(subdir)
-        self.fetcher = fetcher(self.tests)
+        self.fetcher: StatFetcher
+        # self.fetcher = fetcher(self.tests)
 
     def add_time(self, test_file: Path, run_time: int) -> None:
         # getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother
@@ -99,8 +110,8 @@ class TestPicker:
                     break
             if test_name is not None:
                 break
-        assert test_name is not None
+        # assert test_name is not None
-        self.fetcher.add_run_time(test_name, run_time)
+        # self.fetcher.add_run_time(test_name, run_time)
 
     def dump_stats(self) -> str:
         res = array.array('I')
@@ -109,7 +120,8 @@ class TestPicker:
         return base64.standard_b64encode(res.tobytes()).decode('utf-8')
 
     def fetch_stats(self):
-        self.fetcher.read_stats()
+        # self.fetcher.read_stats()
+        pass
 
     def load_stats(self, serialized: str):
         times = array.array('I')
@@ -121,7 +133,7 @@ class TestPicker:
             idx += 1
 
     def parse_txt(self, path: Path):
-        if self.include_files_regex.match(str(path)) is None or self.exclude_files_regex.match(str(path)) is not None:
+        if self.include_files_regex.search(str(path)) is None or self.exclude_files_regex.search(str(path)) is not None:
             return
         with path.open('r') as f:
             test_name: str | None = None
@@ -151,8 +163,8 @@ class TestPicker:
                 test_class = test_name
             if priority is None:
                 priority = 1.0
-            if self.include_tests_regex.match(test_class) is None \
+            if self.include_tests_regex.search(test_class) is None \
-                    or self.exclude_tests_regex.match(test_class) is not None:
+                    or self.exclude_tests_regex.search(test_class) is not None:
                 return
             if test_class not in self.tests:
                 self.tests[test_class] = TestDescription(path, test_class, priority)
@@ -193,9 +205,10 @@ class TestPicker:
                 candidates = [v]
             elif this_time == min_runtime:
                 candidates.append(v)
-        choice = random.randint(0, len(candidates) - 1)
+        candidates.sort()
+        choice = config.random.randint(0, len(candidates) - 1)
         test = candidates[choice]
-        result = test.paths[random.randint(0, len(test.paths) - 1)]
+        result = test.paths[config.random.randint(0, len(test.paths) - 1)]
         if self.restart_test.match(result.name):
             return self.list_restart_files(result)
         else:
@@ -206,7 +219,7 @@ class OldBinaries:
     def __init__(self):
         self.first_file_expr = re.compile(r'.*-1\.(txt|toml)')
         self.old_binaries_path: Path = config.old_binaries_path
-        self.binaries: OrderedDict[version.Version, Path] = collections.OrderedDict()
+        self.binaries: OrderedDict[Version, Path] = collections.OrderedDict()
         if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir():
             return
         exec_pattern = re.compile(r'fdbserver-\d+\.\d+\.\d+(\.exe)?')
@@ -220,13 +233,14 @@ class OldBinaries:
         version_str = file.name.split('-')[1]
         if version_str.endswith('.exe'):
             version_str = version_str[0:-len('.exe')]
-        self.binaries[version.Version.parse(version_str)] = file
+        ver = Version.parse(version_str)
+        self.binaries[ver] = file
 
     def choose_binary(self, test_file: Path) -> Path:
         if len(self.binaries) == 0:
             return config.binary
-        max_version = version.Version.max_version()
+        max_version = Version.max_version()
-        min_version = version.Version.parse('5.0.0')
+        min_version = Version.parse('5.0.0')
         dirs = test_file.parent.parts
         if 'restarting' not in dirs:
             return config.binary
@@ -239,16 +253,16 @@ class OldBinaries:
             # upgrade test -- we only return an old version for the first test file
             return config.binary
         if version_expr[0] == 'from' or version_expr[0] == 'to':
-            min_version = version.Version.parse(version_expr[1])
+            min_version = Version.parse(version_expr[1])
         if len(version_expr) == 4 and version_expr[2] == 'until':
-            max_version = version.Version.parse(version_expr[3])
+            max_version = Version.parse(version_expr[3])
         candidates: List[Path] = []
         for ver, binary in self.binaries.items():
             if min_version <= ver <= max_version:
                 candidates.append(binary)
         if len(candidates) == 0:
             return config.binary
-        return random.choice(candidates)
+        return config.random.choice(candidates)
 
 
 def is_restarting_test(test_file: Path):
@@ -287,7 +301,7 @@ class ResourceMonitor(threading.Thread):
 class TestRun:
     def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID,
                  restarting: bool = False, test_determinism: bool = False,
-                 stats: str | None = None, expected_unseed: int | None = None):
+                 stats: str | None = None, expected_unseed: int | None = None, will_restart: bool = False):
         self.binary = binary
         self.test_file = test_file
         self.random_seed = random_seed
@@ -299,13 +313,16 @@ class TestRun:
         self.err_out: str = 'error.xml'
         self.use_valgrind: bool = config.use_valgrind
         self.old_binary_path: Path = config.old_binaries_path
-        self.buggify_enabled: bool = random.random() < config.buggify_on_ratio
+        self.buggify_enabled: bool = config.random.random() < config.buggify_on_ratio
         self.fault_injection_enabled: bool = True
         self.trace_format = config.trace_format
+        if Version.of_binary(self.binary) < "6.1.0":
+            self.trace_format = None
         self.temp_path = config.run_dir / str(self.uid)
         # state for the run
         self.retryable_error: bool = False
-        self.summary: Summary = Summary(binary)
+        self.summary: Summary = Summary(binary, uid=self.uid, stats=self.stats, expected_unseed=self.expected_unseed,
+                                        will_restart=will_restart)
         self.run_time: int = 0
         self.success = self.run()
 
@@ -332,9 +349,11 @@ class TestRun:
         command += [str(self.binary.absolute()),
                     '-r', 'test' if is_no_sim(self.test_file) else 'simulation',
                     '-f', str(self.test_file),
-                    '-s', str(self.random_seed),
+                    '-s', str(self.random_seed)]
-                    '-fi', 'on' if self.fault_injection_enabled else 'off',
+        if self.trace_format is not None:
-                    '--trace_format', self.trace_format]
+            command += ['--trace_format', self.trace_format]
+        if Version.of_binary(self.binary) >= '7.1.0':
+            command += ['-fi', 'on' if self.fault_injection_enabled else 'off']
         if self.restarting:
             command.append('--restarting')
         if self.buggify_enabled:
@@ -347,12 +366,16 @@ class TestRun:
         # self.log_test_plan(out)
         resources = ResourceMonitor()
         resources.start()
-        process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.temp_path)
+        process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=self.temp_path,
+                                   text=True)
         did_kill = False
+        timeout = 20 * config.kill_seconds if self.use_valgrind else config.kill_seconds
+        err_out: str
         try:
-            process.wait(20 * config.kill_seconds if self.use_valgrind else config.kill_seconds)
+            _, err_out = process.communicate(timeout=timeout)
         except subprocess.TimeoutExpired:
             process.kill()
+            _, err_out = process.communicate()
             did_kill = True
         resources.stop()
         resources.join()
@@ -360,10 +383,9 @@ class TestRun:
         self.summary.runtime = resources.time()
         self.summary.max_rss = resources.max_rss
         self.summary.was_killed = did_kill
-        self.summary = Summary(self.binary, runtime=resources.time(), max_rss=resources.max_rss,
+        self.summary.valgrind_out_file = valgrind_file
-                               was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file,
+        self.summary.error_out = err_out
-                               expected_unseed=self.expected_unseed)
+        self.summary.summarize(self.temp_path, ' '.join(command))
-        self.summary.summarize(self.temp_path)
         self.summary.out.dump(sys.stdout)
         return self.summary.ok()
 
@@ -371,19 +393,11 @@ class TestRun:
 class TestRunner:
     def __init__(self):
         self.uid = uuid.uuid4()
-        self.test_path: str = 'tests'
+        self.test_path: Path = Path('tests')
         self.cluster_file: str | None = None
         self.fdb_app_dir: str | None = None
-        self.stat_fetcher: StatFetcherCreator = \
-            lambda x: FileStatsFetcher(os.getenv('JOSHUA_STAT_FILE', 'stats.json'), x)
         self.binary_chooser = OldBinaries()
-
+        self.test_picker = TestPicker(self.test_path)
-    def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str):
-        def fdb_fetcher(tests: OrderedDict[str, TestDescription]):
-            from . import fdb
-            return fdb.FDBStatFetcher(cluster_file, app_dir, tests)
-
-        self.stat_fetcher = fdb_fetcher
 
     def backup_sim_dir(self, seed: int):
         temp_dir = config.run_dir / str(self.uid)
@@ -405,34 +419,37 @@ class TestRunner:
         count = 0
         result: bool = True
         for file in test_files:
+            will_restart = count + 1 < len(test_files)
             binary = self.binary_chooser.choose_binary(file)
-            unseed_check = random.random() < config.unseed_check_ratio
+            unseed_check = config.random.random() < config.unseed_check_ratio
             if unseed_check and count != 0:
                 # for restarting tests we will need to restore the sim2 after the first run
                 self.backup_sim_dir(seed + count - 1)
             run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
-                          stats=test_picker.dump_stats())
+                          stats=test_picker.dump_stats(), will_restart=will_restart)
             result = result and run.success
-            test_picker.add_time(file, run.run_time)
+            test_picker.add_time(test_files[0], run.run_time)
-            if run.success and unseed_check and run.summary.unseed is not None:
-                self.restore_sim_dir(seed + count - 1)
-                run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
-                               stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed)
-                test_picker.add_time(file, run2.run_time)
             if not result:
                 return False
+            if unseed_check and run.summary.unseed is not None:
+                if count != 0:
+                    self.restore_sim_dir(seed + count - 1)
+                run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
+                               stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed,
+                               will_restart=will_restart)
+                test_picker.add_time(file, run2.run_time)
+                result = result and run.success
             count += 1
         return result
 
     def run(self, stats: str | None) -> bool:
-        seed = random.randint(0, 2 ** 32 - 1)
+        seed = config.random.randint(0, 2 ** 32 - 1)
-        test_picker = TestPicker(Path(self.test_path), self.stat_fetcher)
         if stats is not None:
-            test_picker.load_stats(stats)
+            self.test_picker.load_stats(stats)
         else:
-            test_picker.fetch_stats()
+            self.test_picker.fetch_stats()
-        test_files = test_picker.choose_test()
+        test_files = self.test_picker.choose_test()
-        success = self.run_tests(test_files, seed, test_picker)
+        success = self.run_tests(test_files, seed, self.test_picker)
         if config.clean_up:
             shutil.rmtree(config.run_dir / str(self.uid))
         return success

contrib/TestHarness2/test_harness/summarize.py

@@ -13,7 +13,6 @@ import xml.sax.saxutils
 
 from pathlib import Path
 from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple
-from xml.dom import minidom
 
 from test_harness.config import config
 
@@ -46,7 +45,7 @@ class SummaryTree:
         # However, our xml is very simple and therefore serializing manually is easy enough
         attrs = []
         for k, v in self.attributes.items():
-            attrs.append('{}="{}"'.format(k, xml.sax.saxutils.escape(v)))
+            attrs.append('{}={}'.format(k, xml.sax.saxutils.quoteattr(v)))
         elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' '))
         out.write(elem)
         if config.pretty_print:
@@ -92,7 +91,7 @@ class ParseHandler:
         self.out = out
         self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict()
 
-    def add_handler(self, attr: Tuple[str, str], callback: ParserCallback) -> None:
+    def add_handler(self, attr: Tuple[str, Optional[str]], callback: ParserCallback) -> None:
         if attr in self.events:
             self.events[attr].append(callback)
         else:
@@ -319,7 +318,7 @@ class TraceFiles:
                 else:
                     self.timestamps.append(ts)
                     self.runs[ts] = [file]
-        self.timestamps.sort()
+        self.timestamps.sort(reverse=True)
 
     def __getitem__(self, idx: int) -> List[Path]:
         res = self.runs[self.timestamps[idx]]
@@ -333,7 +332,8 @@ class TraceFiles:
 class Summary:
     def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None,
                  was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None,
-                 exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None):
+                 exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None,
+                 error_out: str = None, will_restart: bool = False):
         self.binary = binary
         self.runtime: float = runtime
         self.max_rss: int | None = max_rss
@@ -353,17 +353,21 @@ class Summary:
         self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict()
         self.test_count: int = 0
         self.tests_passed: int = 0
+        self.error_out = error_out
+        self.stderr_severity: str = '40'
+        self.will_restart: bool = will_restart
 
         if uid is not None:
             self.out.attributes['TestUID'] = str(uid)
         if stats is not None:
             self.out.attributes['Statistics'] = stats
         self.out.attributes['JoshuaSeed'] = str(config.joshua_seed)
+        self.out.attributes['WillRestart'] = '1' if self.will_restart else '0'
 
         self.handler = ParseHandler(self.out)
         self.register_handlers()
 
-    def summarize(self, trace_dir: Path):
+    def summarize(self, trace_dir: Path, command: str):
         trace_files = TraceFiles(trace_dir)
         if len(trace_files) == 0:
             self.error = True
@@ -371,14 +375,15 @@ class Summary:
             child = SummaryTree('NoTracesFound')
             child.attributes['Severity'] = '40'
             child.attributes['Path'] = str(trace_dir.absolute())
+            child.attributes['Command'] = command
             self.out.append(child)
+            return
         for f in trace_files[0]:
             self.parse_file(f)
         self.done()
 
     def ok(self):
-        return not self.error and self.tests_passed == self.test_count and self.tests_passed >= 0\
+        return not self.error
-               and self.test_end_found
 
     def done(self):
         if config.print_coverage:
@@ -429,7 +434,38 @@ class Summary:
             child = SummaryTree('TestUnexpectedlyNotFinished')
             child.attributes['Severity'] = '40'
             self.out.append(child)
+        if self.error_out is not None and len(self.error_out) > 0:
+            if self.stderr_severity == '40':
+                self.error = True
+            lines = self.error_out.split('\n')
+            stderr_bytes = 0
+            for line in lines:
+                remaining_bytes = config.max_stderr_bytes - stderr_bytes
+                if remaining_bytes > 0:
+                    out_err = line[0:remaining_bytes] + ('...' if len(line) > remaining_bytes else '')
+                    child = SummaryTree('StdErrOutput')
+                    child.attributes['Severity'] = self.stderr_severity
+                    child.attributes['Output'] = out_err
+                    self.out.append(child)
+                stderr_bytes += len(line)
+            if stderr_bytes > config.max_stderr_bytes:
+                child = SummaryTree('StdErrOutputTruncated')
+                child.attributes['Severity'] = self.stderr_severity
+                child.attributes['BytesRemaining'] = stderr_bytes - config.max_stderr_bytes
+                self.out.append(child)
+
         self.out.attributes['Ok'] = '1' if self.ok() else '0'
+        if not self.ok():
+            reason = 'Unknown'
+            if self.error:
+                reason = 'ProducedErrors'
+            elif not self.test_end_found:
+                reason = 'TestDidNotFinish'
+            elif self.tests_passed == 0:
+                reason = 'NoTestsPassed'
+            elif self.test_count != self.tests_passed:
+                reason = 'Expected {} tests to pass, but only {} did'.format(self.test_count, self.tests_passed)
+            self.out.attributes['FailReason'] = reason
 
     def parse_file(self, file: Path):
         parser: Parser
@@ -581,3 +617,8 @@ class Summary:
                 self.out.append(child)
         self.handler.add_handler(('Type', 'BuggifySection'), buggify_section)
         self.handler.add_handler(('Type', 'FaultInjected'), buggify_section)
+
+        def stderr_severity(attrs: Dict[str, str]):
+            if 'NewSeverity' in attrs:
+                self.stderr_severity = attrs['NewSeverity']
+        self.handler.add_handler(('Type', 'StderrSeverity'), stderr_severity)

contrib/TestHarness2/test_harness/version.py

@@ -1,3 +1,9 @@
+from functools import total_ordering
+from pathlib import Path
+from typing import Tuple
+
+
+@total_ordering
 class Version:
     def __init__(self):
         self.major: int = 0
@@ -7,24 +13,39 @@ class Version:
     def version_tuple(self):
         return self.major, self.minor, self.patch
 
-    def __eq__(self, other):
+    def _compare(self, other) -> int:
-        return self.version_tuple() == other.version_tuple()
+        lhs: Tuple[int, int, int] = self.version_tuple()
+        rhs: Tuple[int, int, int]
+        if isinstance(other, Version):
+            rhs = other.version_tuple()
+        else:
+            rhs = Version.parse(str(other)).version_tuple()
+        if lhs < rhs:
+            return -1
+        elif lhs > rhs:
+            return 1
+        else:
+            return 0
 
-    def __lt__(self, other):
+    def __eq__(self, other) -> bool:
-        return self.version_tuple() < other.version_tuple()
+        return self._compare(other) == 0
 
-    def __le__(self, other):
+    def __lt__(self, other) -> bool:
-        return self.version_tuple() <= other.version_tuple()
+        return self._compare(other) < 0
-
-    def __gt__(self, other):
-        return self.version_tuple() > other.version_tuple()
-
-    def __ge__(self, other):
-        return self.version_tuple() >= other.version_tuple()
 
     def __hash__(self):
         return hash(self.version_tuple())
 
+    def __str__(self):
+        return format('{}.{}.{}'.format(self.major, self.minor, self.patch))
+
+    @staticmethod
+    def of_binary(binary: Path):
+        parts = binary.name.split('-')
+        if len(parts) != 2:
+            return Version.max_version()
+        return Version.parse(parts[1])
+
     @staticmethod
     def parse(version: str):
         version_tuple = version.split('.')
@@ -34,6 +55,7 @@ class Version:
             self.minor = int(version_tuple[1])
             if len(version_tuple) > 2:
                 self.patch = int(version_tuple[2])
+        return self
 
     @staticmethod
     def max_version():
@@ -41,3 +63,4 @@ class Version:
         self.major = 2**32 - 1
         self.minor = 2**32 - 1
         self.patch = 2**32 - 1
+        return self