From c5f623af3198d8292025539775926d825f94d759 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 15 Aug 2022 11:09:08 -0600 Subject: [PATCH 01/29] Add new test harness --- contrib/TestHarness2/.gitignore | 2 + contrib/TestHarness2/stubs/fdb/__init__.pyi | 323 ++++++++++ contrib/TestHarness2/test_harness/__init__.py | 1 + contrib/TestHarness2/test_harness/app.py | 24 + contrib/TestHarness2/test_harness/config.py | 105 ++++ contrib/TestHarness2/test_harness/fdb.py | 40 ++ contrib/TestHarness2/test_harness/run.py | 423 +++++++++++++ .../TestHarness2/test_harness/summarize.py | 571 ++++++++++++++++++ contrib/TestHarness2/test_harness/version.py | 43 ++ 9 files changed, 1532 insertions(+) create mode 100644 contrib/TestHarness2/.gitignore create mode 100644 contrib/TestHarness2/stubs/fdb/__init__.pyi create mode 100644 contrib/TestHarness2/test_harness/__init__.py create mode 100644 contrib/TestHarness2/test_harness/app.py create mode 100644 contrib/TestHarness2/test_harness/config.py create mode 100644 contrib/TestHarness2/test_harness/fdb.py create mode 100644 contrib/TestHarness2/test_harness/run.py create mode 100644 contrib/TestHarness2/test_harness/summarize.py create mode 100644 contrib/TestHarness2/test_harness/version.py diff --git a/contrib/TestHarness2/.gitignore b/contrib/TestHarness2/.gitignore new file mode 100644 index 0000000000..80682f9552 --- /dev/null +++ b/contrib/TestHarness2/.gitignore @@ -0,0 +1,2 @@ +/tmp/ +/venv diff --git a/contrib/TestHarness2/stubs/fdb/__init__.pyi b/contrib/TestHarness2/stubs/fdb/__init__.pyi new file mode 100644 index 0000000000..bf0eedbd7d --- /dev/null +++ b/contrib/TestHarness2/stubs/fdb/__init__.pyi @@ -0,0 +1,323 @@ +from typing import Generic, TypeVar, Callable +from . import fdboptions +from fdboptions import StreamingMode + +def api_version(version: int) -> None: ... + +def open(cluster_file: str = None, event_model: str = None) -> Database: + ... + +T = TypeVar('T') +options = fdboptions.NetworkOptions +StreamingMode = StreamingMode + + +class Future(Generic[T]): + def wait(self) -> T: + ... + + def is_ready(self) -> bool: + ... + + def block_until_ready(self) -> None: + ... + + def on_ready(self, callback: Callable[[Future[T]], None]) -> None: + ... + + def cancel(self): + ... + + @staticmethod + def wait_for_any(*futures: Future): + ... + + +class FutureString(Future[bytes]): + __class__ = property(bytes) + def as_foundationdb_key(self) -> bytes: + ... + + def as_foundationdb_value(self) -> bytes: + ... + + +class ValueType(FutureString): + def present(self) -> bool: + ... + +class KeyType(FutureString): + pass + + +Key = KeyType | bytes +Value = ValueType | bytes +Version = int + +class KVIter: + def __iter__(self) -> KVIter: + ... + + def __next__(self) -> KeyValue: + ... + + +class KeyValue: + key: Key + value: Value + + def __iter__(self) -> KVIter: + ... + + +class KeySelector: + @classmethod + def last_less_than(cls, key: Key) -> KeySelector: + ... + + @classmethod + def last_less_or_equal(cls, key: Key) -> KeySelector: + ... + + @classmethod + def first_greater_than(cls, key: Key) -> KeySelector: + ... + + @classmethod + def first_greater_or_equal(cls, key: Key) -> KeySelector: + ... + + def __add__(self, offset: int) -> KeySelector: + ... + + def __sub__(self, offset: int) -> KeySelector: + ... + + +class Error: + code: int + description: str + + +class Tenant: + def create_transaction(self) -> Transaction: + ... + + +class _SnapshotTransaction: + db: Database + def get(self, key: Key) -> bytes | None: + ... + + def get_key(self, key_selector: KeySelector) -> bytes: + ... + + def __getitem__(self, key: Key | slice) -> bytes: + ... + + def get_range(self, begin: Key, end: Key, limit: int = 0, reverse: bool = False, + streaming_mode: StreamingMode = StreamingMode.exact) -> KeyValue: + ... + + def get_range_startswith(self, prefix: bytes, + limit: int = 0, + reverse: bool = False, + streaming_mode: StreamingMode = StreamingMode.exact): + ... + + +class Transaction(_SnapshotTransaction): + options: fdboptions.TransactionOptions + snapshot: _SnapshotTransaction + + def set(self, key: Key, value: Value) -> None: + ... + + def clear(self, key: Key) -> None: + ... + + def clear_range(self, begin: Key, end: Key) -> None: + ... + + def clear_range_startswith(self, prefix: bytes) -> None: + ... + + def __setitem__(self, key: Key, value: Value) -> None: + ... + + def __delitem__(self, key: Key) -> None: + ... + + def add(self, key: Key, param: bytes) -> None: + ... + + def bit_and(self, key: Key, param: bytes) -> None: + ... + + def bit_or(self, key: Key, param: bytes) -> None: + ... + + def bit_xor(self, key: Key, param: bytes) -> None: + ... + + def max(self, key: Key, param: bytes) -> None: + ... + + def byte_max(self, key: Key, param: bytes) -> None: + ... + + def min(self, key: Key, param: bytes) -> None: + ... + + def byte_min(self, key: Key, param: bytes) -> None: + ... + + def set_versionstamped_key(self, key: Key, param: bytes) -> None: + ... + + def set_versionstamped_value(self, key: Key, param: bytes) -> None: + ... + + def commit(self) -> Future[None]: + ... + + def on_error(self, err: Error) -> Future[None]: + ... + + def reset(self) -> None: + ... + + def cancel(self) -> None: + ... + + def watch(self, key: Key) -> Future[None]: + ... + + def add_read_conflict_range(self, begin: Key, end: Key) -> None: + ... + + def add_read_conflict_key(self, key: Key) -> None: + ... + + def add_write_conflict_range(self, begin: Key, end: Key) -> None: + ... + + def add_write_conflict_key(self, key: Key) -> None: + ... + + def set_read_version(self, version: Version) -> None: + ... + + def get_read_version(self) -> Version: + ... + + def get_committed_version(self) -> Version: + ... + + def get_versionstamp(self) -> FutureString: + ... + + +class Database: + options: fdboptions.DatabaseOptions + + def create_transaction(self) -> Transaction: + ... + + def open_tenant(self, tenant_name: str) -> Tenant: + ... + + def get(self, key: Key) -> bytes | None: + ... + + def get_key(self, key_selector: KeySelector) -> bytes: + ... + + def clear(self, ): + + def __getitem__(self, key: Key | slice) -> bytes: + ... + + def __setitem__(self, key: Key, value: Value) -> None: + ... + + def __delitem__(self, key: Key) -> None: + ... + + def get_range(self, begin: Key, end: Key, limit: int = 0, reverse: bool = False, + streaming_mode: StreamingMode = StreamingMode.exact) -> KeyValue: + ... + + +class Subspace: + def __init__(self, **kwargs): + ... + + def key(self) -> bytes: + ... + + def pack(self, tuple: tuple = tuple()) -> bytes: + ... + + def pack_with_versionstamp(self, tuple: tuple) -> bytes: + ... + + def unpack(self, key: bytes) -> tuple: + ... + + def range(self, tuple: tuple = tuple()) -> slice: + ... + + def contains(self, key: bytes) -> bool: + ... + + def subspace(self, tuple: tuple) -> Subspace: + ... + + def __getitem__(self, item) -> Subspace: + ... + + +class DirectoryLayer: + def __init__(self, *kwargs): + ... + + def create_or_open(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: + ... + + def open(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: + ... + + def create(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: + ... + + def move(self, tr: Transaction, old_path: tuple | str, new_path: tuple | str) -> DirectorySubspace: + ... + + def remove(self, tr: Transaction, path: tuple | str) -> None: + ... + + def remove_if_exists(self, tr: Transaction, path: tuple | str) -> None: + ... + + def list(self, tr: Transaction, path: tuple | str = ()) -> List[str]: + ... + + def exists(self, tr: Transaction, path: tuple | str) -> bool: + ... + + def get_layer(self) -> bytes: + ... + + def get_path(self) -> tuple: + ... + + +class DirectorySubspace(DirectoryLayer, Subspace): + def move_to(self, tr: Transaction, new_path: tuple | str) -> DirectorySubspace: + ... + +directory: DirectoryLayer + +def transactional(*tr_args, **tr_kwargs) -> Callable: + ... \ No newline at end of file diff --git a/contrib/TestHarness2/test_harness/__init__.py b/contrib/TestHarness2/test_harness/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/contrib/TestHarness2/test_harness/__init__.py @@ -0,0 +1 @@ + diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py new file mode 100644 index 0000000000..d5e6e0da7f --- /dev/null +++ b/contrib/TestHarness2/test_harness/app.py @@ -0,0 +1,24 @@ +import argparse +import random + +import test_harness.config +from test_harness.config import config +from test_harness.run import TestRunner + + +if __name__ == '__main__': + # seed the random number generator + parser = argparse.ArgumentParser('TestHarness') + test_harness.config.build_arguments(parser) + test_runner = TestRunner() + # initialize arguments + parser.add_argument('--joshua-dir', type=str, help='Where to write FDB data to', required=False) + parser.add_argument('-C', '--cluster-file', type=str, help='Path to fdb cluster file', required=False) + parser.add_argument('--stats', type=str, + help='A base64 encoded list of statistics (used to reproduce runs)', + required=False) + args = parser.parse_args() + test_harness.config.extract_args(args) + random.seed(config.JOSHUA_SEED) + if not test_runner.run(args.stats): + exit(1) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py new file mode 100644 index 0000000000..4659833a3b --- /dev/null +++ b/contrib/TestHarness2/test_harness/config.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import argparse +import copy +import os +import random +from enum import Enum +from pathlib import Path +from typing import List, Any + + +class BuggifyOptionValue(Enum): + ON = 1 + OFF = 2 + RANDOM = 3 + + +class BuggifyOption: + def __init__(self, val: str | None = None): + self.value = BuggifyOptionValue.RANDOM + v = val.lower() + if v == 'on' or v == '1' or v == 'true': + self.value = BuggifyOptionValue.ON + elif v == 'off' or v == '0' or v == 'false': + self.value = BuggifyOptionValue.OFF + + +class ConfigValue: + def __init__(self, name: str, **kwargs): + self.name = name + self.value = None + self.kwargs = kwargs + if 'default' in self.kwargs: + self.value = self.kwargs['default'] + + def get_arg_name(self) -> str: + if 'long_name' in self.kwargs: + return self.kwargs['long_name'] + else: + return self.name + + def add_to_args(self, parser: argparse.ArgumentParser): + kwargs = copy.copy(self.kwargs) + long_name = self.name + short_name = None + if 'long_name' in kwargs: + long_name = kwargs['long_name'] + del kwargs['long_name'] + if 'short_name' in kwargs: + short_name = kwargs['short_name'] + del kwargs['short_name'] + long_name = long_name.replace('_', '-') + if short_name is None: + parser.add_argument('--{}'.format(long_name), **kwargs) + else: + parser.add_argument('-{}'.format(short_name), '--{}'.format(long_name), **kwargs) + + def get_value(self, args: argparse.Namespace) -> tuple[str, Any]: + return self.name, args.__getattribute__(self.get_arg_name()) + + +configuration: List[ConfigValue] = [ + ConfigValue('kill_seconds', default=60 * 30, help='Timeout for individual test', type=int), + ConfigValue('buggify_on_ratio', default=0.8, help='Probability that buggify is turned on', type=float), + ConfigValue('write_run_times', default=False, help='Write back probabilities after each test run', + action='store_true'), + ConfigValue('unseed_check_ratio', default=0.05, help='Probability for doing determinism check', type=float), + ConfigValue('test_dirs', default=['slow', 'fast', 'restarting', 'rare', 'noSim'], nargs='*'), + ConfigValue('trace_format', default='json', choices=['json', 'xml']), + ConfigValue('crash_on_error', long_name='no_crash', default=True, action='store_false'), + ConfigValue('max_warnings', default=10, short_name='W', type=int), + ConfigValue('max_errors', default=10, short_name='E', type=int), + ConfigValue('old_binaries_path', default=Path('/opt/joshua/global_data/oldBinaries'), type=Path), + ConfigValue('use_valgrind', default=False, action='store_true'), + ConfigValue('buggify', short_name='b', default=BuggifyOption('random'), type=BuggifyOption, + choices=['on', 'off', 'random']), + ConfigValue('pretty_print', short_name='P', default=False, action='store_true'), + ConfigValue('clean_up', default=True), + ConfigValue('run_dir', default=Path('tmp'), type=Path), + ConfigValue('joshua_seed', default=int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))), type=int), + ConfigValue('print_coverage', default=False, action='store_true'), + ConfigValue('binary', default=Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver'), + help='Path to executable', type=Path), + ConfigValue('output_format', short_name='O', type=str, choices=['json', 'xml'], default='xml'), +] + + +class Config: + def __init__(self): + for val in configuration: + super().__setattr__(val.name.upper(), val.value) + + +config = Config() + + +def build_arguments(parser: argparse.ArgumentParser): + for val in configuration: + val.add_to_args(parser) + + +def extract_args(args: argparse.Namespace): + for val in configuration: + k, v = val.get_value(args) + config.__setattr__(k.upper(), v) diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py new file mode 100644 index 0000000000..ac1de09989 --- /dev/null +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import OrderedDict + +import fdb +import struct + +from run import StatFetcher, TestDescription + + +class FDBStatFetcher(StatFetcher): + def __init__(self, cluster_file: str | None, app_dir: str, tests: OrderedDict[str, TestDescription]): + super().__init__(tests) + fdb.api_version(630) + self.db: fdb.Database = fdb.open(cluster_file) + self.stats_dir: fdb.DirectorySubspace = self.open_stats_dir(self.db, app_dir) + + @fdb.transactional + def open_stats_dir(self, tr, app_dir: str) -> fdb.DirectorySubspace: + app_dir_path = app_dir.split(',') + app_dir_path.append('runtime_stats') + return fdb.directory.create_or_open(tr, tuple(app_dir_path)) + + @fdb.transactional + def read_stats_from_db(self, tr): + for k, v in tr[self.stats_dir.range()]: + test_name = self.stats_dir.unpack(k)[0] + if test_name in self.tests.keys(): + self.tests[test_name] = struct.unpack(' None: + # getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother + test_name = None + for name, test in self.tests.items(): + for p in test.paths: + if p.absolute() == test_file.absolute(): + test_name = name + break + if test_name is not None: + break + assert test_name is not None + self.fetcher.add_run_time(test_name, run_time) + + def dump_stats(self) -> str: + res = array.array('I') + for _, spec in self.tests.items(): + res.append(spec.total_runtime) + return base64.standard_b64encode(res.tobytes()).decode('utf-8') + + def fetch_stats(self): + self.fetcher.read_stats() + + def load_stats(self, serialized: str): + times = array.array('I') + times.frombytes(base64.standard_b64decode(serialized)) + assert len(times) == len(self.tests.items()) + idx = 0 + for _, spec in self.tests.items(): + spec.total_runtime = times[idx] + idx += 1 + + def parse_txt(self, path: Path): + with path.open('r') as f: + test_name: str | None = None + test_class: str | None = None + priority: float | None = None + for line in f: + line = line.strip() + kv = line.split('=') + if len(kv) != 2: + continue + kv[0] = kv[0].strip() + kv[1] = kv[1].strip(' \r\n\t\'"') + if kv[0] == 'testTitle' and test_name is None: + test_name = kv[1] + if kv[0] == 'testClass' and test_class is None: + test_class = kv[1] + if kv[0] == 'testPriority' and priority is None: + try: + priority = float(kv[1]) + except ValueError: + pass + if test_name is not None and test_class is not None and priority is not None: + break + if test_name is None: + return + if test_class is None: + test_class = test_name + if priority is None: + priority = 1.0 + if test_class not in self.tests: + self.tests[test_class] = TestDescription(path, test_class, priority) + else: + self.tests[test_class].paths.append(path) + + def walk_test_dir(self, test: Path): + if test.is_dir(): + for file in test.iterdir(): + self.walk_test_dir(file) + else: + # check whether we're looking at a restart test + if self.follow_test.match(test.name) is not None: + return + if test.suffix == '.txt' or test.suffix == '.toml': + self.parse_txt(test) + + @staticmethod + def list_restart_files(start_file: Path) -> List[Path]: + name = re.sub(r'-\d+.(txt|toml)', '', start_file.name) + res: List[Path] = [] + for test_file in start_file.parent.iterdir(): + if test_file.name.startswith(name): + res.append(test_file) + assert len(res) > 1 + res.sort() + return res + + def choose_test(self) -> List[Path]: + min_runtime: float | None = None + candidates: List[TestDescription] = [] + for _, v in self.tests.items(): + this_time = v.total_runtime * v.priority + if min_runtime is None: + min_runtime = this_time + if this_time < min_runtime: + min_runtime = this_time + candidates = [v] + elif this_time == min_runtime: + candidates.append(v) + choice = random.randint(0, len(candidates) - 1) + test = candidates[choice] + result = test.paths[random.randint(0, len(test.paths) - 1)] + if self.restart_test.match(result.name): + return self.list_restart_files(result) + else: + return [result] + + +class OldBinaries: + def __init__(self): + self.first_file_expr = re.compile(r'.*-1\.(txt|toml)') + self.old_binaries_path: Path = config.OLD_BINARIES_PATH + self.binaries: OrderedDict[version.Version, Path] = collections.OrderedDict() + if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir(): + return + exec_pattern = re.compile(r'fdbserver-\d+\.\d+\.\d+(\.exe)?') + for file in self.old_binaries_path.iterdir(): + if not file.is_file() or not os.access(file, os.X_OK): + continue + if exec_pattern.fullmatch(file.name) is not None: + self._add_file(file) + + def _add_file(self, file: Path): + version_str = file.name.split('-')[1] + if version_str.endswith('.exe'): + version_str = version_str[0:-len('.exe')] + self.binaries[version.Version.parse(version_str)] = file + + def choose_binary(self, test_file: Path) -> Path: + if len(self.binaries) == 0: + return config.BINARY + max_version = version.Version.max_version() + min_version = version.Version.parse('5.0.0') + dirs = test_file.parent.parts + if 'restarting' not in dirs: + return config.BINARY + version_expr = dirs[-1].split('_') + first_file = self.first_file_expr.match(test_file.name) is not None + if first_file and version_expr[0] == 'to': + # downgrade test -- first binary should be current one + return config.BINARY + if not first_file and version_expr[0] == 'from': + # upgrade test -- we only return an old version for the first test file + return config.BINARY + if version_expr[0] == 'from' or version_expr[0] == 'to': + min_version = version.Version.parse(version_expr[1]) + if len(version_expr) == 4 and version_expr[2] == 'until': + max_version = version.Version.parse(version_expr[3]) + candidates: List[Path] = [] + for ver, binary in self.binaries.items(): + if min_version <= ver <= max_version: + candidates.append(binary) + return random.choice(candidates) + + +def is_restarting_test(test_file: Path): + for p in test_file.parts: + if p == 'restarting': + return True + return False + + +def is_no_sim(test_file: Path): + return test_file.parts[-2] == 'noSim' + + +class ResourceMonitor(threading.Thread): + def __init__(self): + super().__init__() + self.start_time = time.time() + self.end_time: float | None = None + self._stop_monitor = False + self.max_rss = 0 + + def run(self) -> None: + while not self._stop_monitor: + time.sleep(1) + resources = resource.getrusage(resource.RUSAGE_CHILDREN) + self.max_rss = max(resources.ru_maxrss, self.max_rss) + + def stop(self): + self.end_time = time.time() + self._stop_monitor = True + + def time(self): + return self.end_time - self.start_time + + +class TestRun: + def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID, + restarting: bool = False, test_determinism: bool = False, + stats: str | None = None, expected_unseed: int | None = None): + self.binary = binary + self.test_file = test_file + self.random_seed = random_seed + self.uid = uid + self.restarting = restarting + self.test_determinism = test_determinism + self.stats: str | None = stats + self.expected_unseed: int | None = expected_unseed + self.err_out: str = 'error.xml' + self.use_valgrind: bool = config.USE_VALGRIND + self.old_binary_path: str = config.OLD_BINARIES_PATH + self.buggify_enabled: bool = random.random() < config.BUGGIFY_ON_RATIO + self.fault_injection_enabled: bool = True + self.trace_format = config.TRACE_FORMAT + # state for the run + self.retryable_error: bool = False + self.summary: Summary | None = None + self.run_time: int = 0 + + def log_test_plan(self, out: SummaryTree): + test_plan: SummaryTree = SummaryTree('TestPlan') + test_plan.attributes['TestUID'] = str(uuid) + test_plan.attributes['RandomSeed'] = str(self.random_seed) + test_plan.attributes['TestFile'] = str(self.test_file) + test_plan.attributes['Buggify'] = '1' if self.buggify_enabled else '0' + test_plan.attributes['FaultInjectionEnabled'] = '1' if self.fault_injection_enabled else '0' + test_plan.attributes['DeterminismCheck'] = '1' if self.test_determinism else '0' + out.append(test_plan) + + def run(self): + command: List[str] = [] + valgrind_file: Path | None = None + if self.use_valgrind: + command.append('valgrind') + valgrind_file = Path('valgrind-{}.xml'.format(self.random_seed)) + dbg_path = os.getenv('FDB_VALGRIND_DBGPATH') + if dbg_path is not None: + command.append('--extra-debuginfo-path={}'.format(dbg_path)) + command += ['--xml=yes', '--xml-file={}'.format(valgrind_file), '-q'] + command += [str(self.binary.absolute()), + '-r', 'test' if is_no_sim(self.test_file) else 'simulation', + '-f', str(self.test_file), + '-s', str(self.random_seed), + '-fi', 'on' if self.fault_injection_enabled else 'off', + '--trace_format', self.trace_format] + if self.restarting: + command.append('--restarting') + if self.buggify_enabled: + command += ['-b', 'on'] + if config.CRASH_ON_ERROR: + command.append('--crash') + temp_path = config.RUN_DIR / str(self.uid) + temp_path.mkdir(parents=True, exist_ok=True) + + # self.log_test_plan(out) + resources = ResourceMonitor() + resources.start() + process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=temp_path) + did_kill = False + try: + process.wait(20 * config.KILL_SECONDS if self.use_valgrind else config.KILL_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + did_kill = True + resources.stop() + resources.join() + self.run_time = round(resources.time()) + self.summary = Summary(self.binary, temp_path, runtime=resources.time(), max_rss=resources.max_rss, + was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file, + expected_unseed=self.expected_unseed) + return self.summary.ok() + + +class TestRunner: + def __init__(self): + self.uid = uuid.uuid4() + self.test_path: str = 'tests' + self.cluster_file: str | None = None + self.fdb_app_dir: str | None = None + self.stat_fetcher: StatFetcherCreator = \ + lambda x: FileStatsFetcher(os.getenv('JOSHUA_STAT_FILE', 'stats.json'), x) + self.binary_chooser = OldBinaries() + + def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str): + def fdb_fetcher(tests: OrderedDict[str, TestDescription]): + from . import fdb + self.stat_fetcher = fdb.FDBStatFetcher(cluster_file, app_dir, tests) + + self.stat_fetcher = fdb_fetcher + + def backup_sim_dir(self, seed: int): + temp_dir = config.RUN_DIR / str(self.uid) + src_dir = temp_dir / 'simfdb' + assert src_dir.is_dir() + dest_dir = temp_dir / 'simfdb.{}'.format(seed) + assert not dest_dir.exists() + shutil.copytree(src_dir, dest_dir) + + def restore_sim_dir(self, seed: int): + temp_dir = config.RUN_DIR / str(self.uid) + src_dir = temp_dir / 'simfdb.{}'.format(seed) + assert src_dir.exists() + dest_dir = temp_dir / 'simfdb' + shutil.rmtree(dest_dir) + shutil.move(src_dir, dest_dir) + + def run_tests(self, test_files: List[Path], seed: int, test_picker: TestPicker) -> bool: + count = 0 + for file in test_files: + binary = self.binary_chooser.choose_binary(file) + unseed_check = random.random() < config.UNSEED_CHECK_RATIO + if unseed_check and count != 0: + # for restarting tests we will need to restore the sim2 after the first run + self.backup_sim_dir(seed + count - 1) + run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, + stats=test_picker.dump_stats()) + success = run.run() + test_picker.add_time(file, run.run_time) + if success and unseed_check and run.summary.unseed is not None: + self.restore_sim_dir(seed + count - 1) + run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, + stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed) + success = run2.run() + test_picker.add_time(file, run2.run_time) + run2.summary.out.dump(sys.stdout) + run.summary.out.dump(sys.stdout) + if not success: + return False + count += 1 + return True + + def run(self, stats: str | None) -> bool: + seed = random.randint(0, 2 ** 32 - 1) + # unseed_check: bool = random.random() < Config.UNSEED_CHECK_RATIO + test_picker = TestPicker(Path(self.test_path), self.stat_fetcher) + if stats is not None: + test_picker.load_stats(stats) + else: + test_picker.fetch_stats() + test_files = test_picker.choose_test() + success = self.run_tests(test_files, seed, test_picker) + if config.CLEAN_UP: + shutil.rmtree(config.RUN_DIR / str(self.uid)) + return success diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py new file mode 100644 index 0000000000..3b5667434a --- /dev/null +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -0,0 +1,571 @@ +from __future__ import annotations + +import collections +import inspect +import json +import re +import sys +import traceback +import uuid +import xml.sax +import xml.sax.handler +import xml.sax.saxutils + +from pathlib import Path +from typing import List, Dict, TextIO, Callable, Optional, OrderedDict +from xml.dom import minidom + +from test_harness.config import config + + +class SummaryTree: + def __init__(self, name: str): + self.name = name + self.children: List[SummaryTree] = [] + self.attributes: Dict[str, str] = {} + + def append(self, element: SummaryTree): + self.children.append(element) + + def to_dict(self) -> Dict: + res = {'Type': self.name} + for k, v in self.attributes.items(): + res[k] = v + children = [] + for child in self.children: + children.append(child.to_dict()) + if len(children) > 0: + res['children'] = children + return res + + def to_json(self, out: TextIO): + json.dump(self.to_dict(), out, indent=(' ' if config.PRETTY_PRINT else None)) + + def append_xml_element(self, doc: minidom.Document, root=True, parent: minidom.Element | None = None): + assert root or parent is not None + this: minidom.Element = doc.createElement(self.name) + for k, v in self.attributes: + this.setAttribute(k, v) + for child in self.children: + child.append_xml_element(doc, root=False, parent=this) + if not root: + parent.appendChild(this) + + def to_xml(self, out: TextIO, prefix: str = ''): + # minidom doesn't support omitting the xml declaration which is a problem for joshua + # However, our xml is very simple and therefore serializing manually is easy enough + attrs = [] + for k, v in self.attributes.items(): + attrs.append('{}="{}"'.format(k, xml.sax.saxutils.escape(v))) + elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' ')) + out.write(elem) + if config.PRETTY_PRINT: + curr_line_len = len(elem) + for i in range(len(attrs)): + attr_len = len(attrs[i]) + if i == 0 or attr_len + curr_line_len + 1 <= 120: + if i != 0: + out.write(' ') + out.write(attrs[i]) + curr_line_len += attr_len + else: + out.write('\n') + out.write(' ' * len(elem)) + out.write(attrs[i]) + curr_line_len = len(elem) + attr_len + else: + out.write(' '.join(attrs)) + if len(self.children) == 0: + out.write('/>') + else: + out.write('>') + for child in self.children: + if config.PRETTY_PRINT: + out.write('\n') + child.to_xml(out, prefix=(' {}'.format(prefix) if config.PRETTY_PRINT else prefix)) + if len(self.children) > 0: + out.write('{}'.format(('\n' if config.PRETTY_PRINT else ''), self.name)) + + def dump(self, out: TextIO): + if config.OUTPUT_FORMAT == 'json': + self.to_json(out) + else: + self.to_xml(out) + + +ParserCallback = Callable[[Optional[Dict[str, Optional[str]]]], Optional[str]] + + +class ParseHandler: + def __init__(self, out: SummaryTree): + self.out = out + self.events: OrderedDict[(str, str), List[ParserCallback]] = collections.OrderedDict() + + def add_handler(self, attr: (str, str), callback: ParserCallback) -> None: + if attr in self.events: + self.events[attr].append(callback) + else: + self.events[attr] = [callback] + + def _call(self, callback: ParserCallback, attrs: Dict[str, str]) -> str | None: + try: + return callback(attrs) + except Exception as e: + _, _, exc_traceback = sys.exc_info() + child = SummaryTree('NonFatalParseError') + child.attributes['Severity'] = '30' + child.attributes['ErrorMessage'] = str(e) + child.attributes['Trace'] = repr(traceback.format_tb(exc_traceback)) + self.out.append(child) + return None + + def handle(self, attrs: Dict[str, str]): + if None in self.events: + for callback in self.events[None]: + self._call(callback, attrs) + for k, v in attrs.items(): + if (k, None) in self.events: + for callback in self.events[(k, None)]: + remap = self._call(callback, attrs) + if remap is not None: + v = remap + attrs[k] = v + if (k, v) in self.events: + for callback in self.events[(k, v)]: + remap = self._call(callback, attrs) + if remap is not None: + v = remap + attrs[k] = v + + +class Parser: + def parse(self, file: TextIO, handler: ParseHandler) -> None: + pass + + +class XmlParser(Parser, xml.sax.handler.ContentHandler): + def __init__(self): + super().__init__() + self.handler: ParseHandler | None = None + + def parse(self, file: TextIO, handler: ParseHandler) -> None: + xml.sax.parse(file, self) + + def startElement(self, name, attrs) -> None: + attributes: Dict[str, str] = {} + for name in attrs.getNames(): + attributes[name] = attrs.getValue(name) + self.handler.handle(attributes) + + +class JsonParser(Parser): + def __init__(self): + super().__init__() + + def parse(self, file: TextIO, handler: ParseHandler): + for line in file: + obj = json.loads(line) + handler.handle(obj) + + +def format_test_error(attrs: Dict[str, str], include_details: bool) -> str: + trace_type = attrs['Type'] + res = trace_type + if trace_type == 'InternalError': + res = '{} {} {}'.format(trace_type, attrs['File'], attrs['Line']) + elif trace_type == 'TestFailure': + res = '{} {}'.format(trace_type, attrs['Reason']) + elif trace_type == 'ValgrindError': + res = '{} {}'.format(trace_type, attrs['What']) + elif trace_type == 'ExitCode': + res = '{0} 0x{1:x}'.format(trace_type, int(attrs['Code'])) + elif trace_type == 'StdErrOutput': + res = '{}: {}'.format(trace_type, attrs['Output']) + elif trace_type == 'BTreeIntegrityCheck': + res = '{}: {}'.format(trace_type, attrs['ErrorDetail']) + for k in ['Error', 'WinErrorCode', 'LinuxErrorCode']: + if k in attrs: + res += ' {}'.format(attrs[k]) + if 'Status' in attrs: + res += ' Status={}'.format(attrs['Status']) + if 'In' in attrs: + res += ' In {}'.format(attrs['In']) + if 'SQLiteError' in attrs: + res += ' SQLiteError={0}({1})'.format(attrs['SQLiteError'], attrs['SQLiteErrorCode']) + if 'Details' in attrs and include_details: + res += ': {}'.format(attrs['Details']) + return res + + +class ValgrindError: + def __init__(self, what: str = '', kind: str = ''): + self.what: str = what + self.kind: str = kind + + def __str__(self): + return 'ValgrindError(what="{}", kind="{}")'.format(self.what, self.kind) + + +class ValgrindHandler(xml.sax.handler.ContentHandler): + def __init__(self): + super().__init__() + self.stack: List[ValgrindError] = [] + self.result: List[ValgrindError] = [] + self.in_kind = False + self.in_what = False + + @staticmethod + def from_content(content): + if isinstance(content, bytes): + return content.decode() + assert isinstance(content, str) + return content + + def characters(self, content): + if len(self.stack) == 0: + return + elif self.in_kind: + self.stack[-1].kind += self.from_content(content) + elif self.in_what: + self.stack[-1].what += self.from_content(content) + + def startElement(self, name, attrs): + if name == 'error': + self.stack.append(ValgrindError()) + if len(self.stack) == 0: + return + if name == 'kind': + self.in_kind = True + elif name == 'what': + self.in_what = True + + def endElement(self, name): + if name == 'error': + self.result.append(self.stack.pop()) + elif name == 'kind': + self.in_kind = False + elif name == 'what': + self.in_what = False + + +def parse_valgrind_output(valgrind_out_file: Path) -> List[str]: + res: List[str] = [] + handler = ValgrindHandler() + with valgrind_out_file.open('r') as f: + xml.sax.parse(f, handler) + for err in handler.result: + if err.kind.startswith('Leak'): + continue + res.append(err.kind) + return res + + +class Coverage: + def __init__(self, file: str, line: str | int, comment: str | None = None): + self.file = file + self.line = int(line) + self.comment = comment + + def __eq__(self, other: Coverage) -> bool: + return self.file == other.file and self.line == other.line and self.comment == other.comment + + def __lt__(self, other: Coverage) -> bool: + if self.file != other.file: + return self.file < other.file + elif self.line != other.line: + return self.line < other.line + elif self.comment != other.comment: + if self.comment is None: + return True + elif other.comment is None: + return False + else: + return self.comment < other.comment + else: + return False + + def __le__(self, other: Coverage) -> bool: + return self.__eq__(other) or self.__lt__(other) + + def __gt__(self, other: Coverage) -> bool: + return not self.__le__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + def __hash__(self): + return hash((self.file, self.line, self.comment)) + + +class TraceFiles: + def __init__(self, path: Path): + self.path: Path = path + self.timestamps: List[int] = [] + self.runs: OrderedDict[int, List[Path]] = collections.OrderedDict() + trace_expr = re.compile(r'trace.*\.(json|xml)') + for file in self.path.iterdir(): + if file.is_file() and trace_expr.match(file.name) is not None: + ts = int(file.name.split('.')[6]) + if ts in self.runs: + self.runs[ts].append(file) + else: + self.timestamps.append(ts) + self.runs[ts] = [file] + self.timestamps.sort() + + def __getitem__(self, idx: int) -> List[Path]: + res = self.runs[self.timestamps[idx]] + res.sort() + return res + + def __len__(self) -> int: + return len(self.runs) + + +class Summary: + def __init__(self, binary: Path, trace_dir: Path, runtime: float = 0, max_rss: int | None = None, + was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None, + exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None): + self.binary = binary + self.runtime: float = runtime + self.max_rss: int | None = max_rss + self.was_killed: bool = was_killed + self.expected_unseed: int | None = expected_unseed + self.exit_code: int = exit_code + self.out: SummaryTree = SummaryTree('Test') + self.test_begin_found: bool = False + self.test_end_found: bool = False + self.unseed: int | None = None + self.valgrind_out_file: Path | None = valgrind_out_file + self.severity_map: OrderedDict[tuple[str, int], int] = collections.OrderedDict() + self.error: bool = False + self.errors: int = 0 + self.error_list: List[str] = [] + self.warnings: int = 0 + self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict() + self.test_count: int = 0 + self.tests_passed: int = 0 + + if uid is not None: + self.out.attributes['TestUID'] = str(uid) + if stats is not None: + self.out.attributes['Statistics'] = stats + self.out.attributes['JoshuaSeed'] = str(config.JOSHUA_SEED) + + self.handler = ParseHandler(self.out) + self.register_handlers() + + trace_files = TraceFiles(trace_dir) + if len(trace_files) == 0: + self.error = True + self.error_list.append('No traces produced') + child = SummaryTree('NoTracesFound') + child.attributes['Severity'] = '40' + child.attributes['Path'] = str(trace_dir.absolute()) + self.out.append(child) + for f in trace_files[0]: + self.parse_file(f) + self.done() + + def ok(self): + return not self.error and self.tests_passed == self.test_count and self.tests_passed >= 0\ + and self.test_end_found + + def done(self): + if config.PRINT_COVERAGE: + for k, v in self.coverage.items(): + child = SummaryTree('CodeCoverage') + child.attributes['File'] = k.file + child.attributes['Line'] = str(k.line) + if not v: + child.attributes['Covered'] = '0' + if k.comment is not None and len(k.comment): + child.attributes['Comment'] = k.comment + self.out.append(child) + if self.warnings > config.MAX_WARNINGS: + child = SummaryTree('WarningLimitExceeded') + child.attributes['Severity'] = '30' + child.attributes['WarningCount'] = str(self.warnings) + self.out.append(child) + if self.errors > config.MAX_ERRORS: + child = SummaryTree('ErrorLimitExceeded') + child.attributes['Severity'] = '40' + child.attributes['ErrorCount'] = str(self.warnings) + self.out.append(child) + self.error_list.append('ErrorLimitExceeded') + if self.was_killed: + child = SummaryTree('ExternalTimeout') + child.attributes['Severity'] = '40' + self.out.append(child) + self.error = True + if self.max_rss is not None: + self.out.attributes['PeakMemory'] = str(self.max_rss) + if self.valgrind_out_file is not None: + try: + whats = parse_valgrind_output(self.valgrind_out_file) + for what in whats: + self.error = True + child = SummaryTree('ValgrindError') + child.attributes['Severity'] = '40' + child.attributes['What'] = what + self.out.append(child) + except Exception as e: + self.error = True + child = SummaryTree('ValgrindParseError') + child.attributes['Severity'] = '40' + child.attributes['ErrorMessage'] = str(e) + self.out.append(child) + self.error_list.append('Failed to parse valgrind output: {}'.format(str(e))) + if not self.test_end_found: + child = SummaryTree('TestUnexpectedlyNotFinished') + child.attributes['Severity'] = '40' + self.out.append(child) + self.out.attributes['Ok'] = '1' if self.ok() else '0' + + def parse_file(self, file: Path): + if file.suffix == '.json': + parser = JsonParser() + elif file.suffix == '.xml': + parser = XmlParser() + else: + child = SummaryTree('TestHarnessBug') + child.attributes['File'] = __file__ + child.attributes['Line'] = str(inspect.getframeinfo(inspect.currentframe()).lineno) + child.attributes['Details'] = 'Unexpected suffix {} for file {}'.format(file.suffix, file.name) + self.error = True + self.out.append(child) + return + with file.open('r') as f: + try: + parser.parse(f, self.handler) + except Exception as e: + child = SummaryTree('SummarizationError') + child.attributes['Severity'] = '40' + child.attributes['ErrorMessage'] = str(e) + self.out.append(child) + self.error_list.append('SummarizationError {}'.format(str(e))) + + def register_handlers(self): + def remap_event_severity(attrs): + if 'Type' not in attrs or 'Severity' not in attrs: + return None + k = (attrs['Type'], int(attrs['Severity'])) + if k in self.severity_map: + return str(self.severity_map[k]) + + self.handler.add_handler(('Severity', None), remap_event_severity) + + def program_start(attrs: Dict[str, str]): + if self.test_begin_found: + return + self.test_begin_found = True + self.out.attributes['RandomSeed'] = attrs['RandomSeed'] + self.out.attributes['SourceVersion'] = attrs['SourceVersion'] + self.out.attributes['Time'] = attrs['ActualTime'] + self.out.attributes['BuggifyEnabled'] = attrs['BuggifyEnabled'] + self.out.attributes['DeterminismCheck'] = '0' if self.expected_unseed is None else '1' + if self.binary.name != 'fdbserver': + self.out.attributes['OldBinary'] = self.binary.name + if 'FaultInjectionEnabled' in attrs: + self.out.attributes['FaultInjectionEnabled'] = attrs['FaultInjectionEnabled'] + + self.handler.add_handler(('Type', 'ProgramStart'), program_start) + + def set_test_file(attrs: Dict[str, str]): + test_file = Path(attrs['TestFile']) + cwd = Path('.').absolute() + try: + test_file = test_file.relative_to(cwd) + except ValueError: + pass + self.out.attributes['TestFile'] = str(test_file) + + self.handler.add_handler(('Type', 'Simulation'), set_test_file) + self.handler.add_handler(('Type', 'NonSimulationTest'), set_test_file) + + def set_elapsed_time(attrs: Dict[str, str]): + if self.test_end_found: + return + self.test_end_found = True + self.unseed = int(attrs['RandomUnseed']) + if self.expected_unseed is not None and self.unseed != self.expected_unseed: + severity = 40 if ('UnseedMismatch', 40) not in self.severity_map \ + else self.severity_map[('UnseedMismatch', 40)] + if severity >= 30: + child = SummaryTree('UnseedMismatch') + child.attributes['Unseed'] = str(self.unseed) + child.attributes['ExpectedUnseed'] = str(self.expected_unseed) + child.attributes['Severity'] = str(severity) + if severity >= 40: + self.error = True + self.error_list.append('UnseedMismatch') + self.out.append(child) + self.out.attributes['SimElapsedTime'] = attrs['SimTime'] + self.out.attributes['RealElapsedTime'] = attrs['RealTime'] + if self.unseed is not None: + self.out.attributes['RandomUnseed'] = str(self.unseed) + + self.handler.add_handler(('Type', 'ElapsedTime'), set_elapsed_time) + + def parse_warning(attrs: Dict[str, str]): + self.warnings += 1 + if self.warnings > config.MAX_WARNINGS: + return + child = SummaryTree(attrs['Type']) + for k, v in attrs.items(): + if k != 'Type': + child.attributes[k] = v + self.out.append(child) + + self.handler.add_handler(('Severity', '30'), parse_warning) + + def parse_error(attrs: Dict[str, str]): + self.errors += 1 + self.error = True + if self.errors > config.MAX_ERRORS: + return + child = SummaryTree(attrs['Type']) + for k, v in attrs.items(): + child.attributes[k] = v + self.out.append(child) + self.error_list.append(format_test_error(attrs, True)) + + self.handler.add_handler(('Severity', '40'), parse_error) + + def coverage(attrs: Dict[str, str]): + covered = True + if 'Covered' in attrs: + covered = int(attrs['Covered']) != 0 + comment = '' + if 'Comment' in attrs: + comment = attrs['Comment'] + c = Coverage(attrs['File'], attrs['Line'], comment) + if covered or c not in self.coverage: + self.coverage[c] = covered + + self.handler.add_handler(('Type', 'CodeCoverage'), coverage) + + def expected_test_pass(attrs: Dict[str, str]): + self.test_count = int(attrs['Count']) + + self.handler.add_handler(('Type', 'TestsExpectedToPass'), expected_test_pass) + + def test_passed(attrs: Dict[str, str]): + if attrs['Passed'] == '1': + self.tests_passed += 1 + + self.handler.add_handler(('Type', 'TestResults'), test_passed) + + def remap_event_severity(attrs: Dict[str, str]): + self.severity_map[(attrs['TargetEvent'], int(attrs['OriginalSeverity']))] = int(attrs['NewSeverity']) + + self.handler.add_handler(('Type', 'RemapEventSeverity'), remap_event_severity) + + def buggify_section(attrs: Dict[str, str]): + if attrs['Type'] == 'FaultInjected' or attrs.get('Activated', '0') == '1': + child = SummaryTree(attrs['Type']) + child.attributes['File'] = attrs['File'] + child.attributes['Line'] = attrs['Line'] + self.out.append(child) + self.handler.add_handler(('Type', 'BuggifySection'), buggify_section) + self.handler.add_handler(('Type', 'FaultInjected'), buggify_section) diff --git a/contrib/TestHarness2/test_harness/version.py b/contrib/TestHarness2/test_harness/version.py new file mode 100644 index 0000000000..b5cc116f37 --- /dev/null +++ b/contrib/TestHarness2/test_harness/version.py @@ -0,0 +1,43 @@ +class Version: + def __init__(self): + self.major: int = 0 + self.minor: int = 0 + self.patch: int = 0 + + def version_tuple(self): + return self.major, self.minor, self.patch + + def __eq__(self, other): + return self.version_tuple() == other.version_tuple() + + def __lt__(self, other): + return self.version_tuple() < other.version_tuple() + + def __le__(self, other): + return self.version_tuple() <= other.version_tuple() + + def __gt__(self, other): + return self.version_tuple() > other.version_tuple() + + def __ge__(self, other): + return self.version_tuple() >= other.version_tuple() + + def __hash__(self): + return hash(self.version_tuple()) + + @staticmethod + def parse(version: str): + version_tuple = version.split('.') + self = Version() + self.major = int(version_tuple[0]) + if len(version_tuple) > 1: + self.minor = int(version_tuple[1]) + if len(version_tuple) > 2: + self.patch = int(version_tuple[2]) + + @staticmethod + def max_version(): + self = Version() + self.major = 2**32 - 1 + self.minor = 2**32 - 1 + self.patch = 2**32 - 1 From dab1c5984d948f8ca31cdbb4e07e6173cd3bcf9c Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 15 Aug 2022 13:56:44 -0600 Subject: [PATCH 02/29] fixed most mypy errors --- contrib/TestHarness2/stubs/fdb/__init__.pyi | 7 +- contrib/TestHarness2/stubs/fdb/fdboptions.pyi | 113 ++++++++++++++++++ contrib/TestHarness2/test_harness/app.py | 7 +- contrib/TestHarness2/test_harness/config.py | 93 +++++++++++--- contrib/TestHarness2/test_harness/fdb.py | 2 +- contrib/TestHarness2/test_harness/run.py | 55 +++++---- .../TestHarness2/test_harness/summarize.py | 104 ++++++++-------- 7 files changed, 281 insertions(+), 100 deletions(-) create mode 100644 contrib/TestHarness2/stubs/fdb/fdboptions.pyi diff --git a/contrib/TestHarness2/stubs/fdb/__init__.pyi b/contrib/TestHarness2/stubs/fdb/__init__.pyi index bf0eedbd7d..4778c7ce92 100644 --- a/contrib/TestHarness2/stubs/fdb/__init__.pyi +++ b/contrib/TestHarness2/stubs/fdb/__init__.pyi @@ -1,5 +1,6 @@ -from typing import Generic, TypeVar, Callable -from . import fdboptions +import fdboptions + +from typing import Generic, TypeVar, Callable, List from fdboptions import StreamingMode def api_version(version: int) -> None: ... @@ -34,7 +35,6 @@ class Future(Generic[T]): class FutureString(Future[bytes]): - __class__ = property(bytes) def as_foundationdb_key(self) -> bytes: ... @@ -234,6 +234,7 @@ class Database: ... def clear(self, ): + ... def __getitem__(self, key: Key | slice) -> bytes: ... diff --git a/contrib/TestHarness2/stubs/fdb/fdboptions.pyi b/contrib/TestHarness2/stubs/fdb/fdboptions.pyi new file mode 100644 index 0000000000..fadfd0ec0c --- /dev/null +++ b/contrib/TestHarness2/stubs/fdb/fdboptions.pyi @@ -0,0 +1,113 @@ +import enum + + +class StreamingMode(enum.Enum): + want_all = -2 + iterator = -1 + exact = 0 + small = 1 + medium = 2 + large = 3 + serial = 4 + +class NetworkOptions(enum.Enum): + local_address = 10 + cluster_file = 20 + trace_enable = 30 + trace_roll_size = 31 + trace_max_logs_size = 32 + trace_log_group = 33 + trace_format = 34 + trace_clock_source = 35 + trace_file_identifier = 36 + trace_partial_file_suffix = 39 + knob = 40 + TLS_plugin = 41 + TLS_cert_bytes = 42 + TLS_cert_path = 43 + TLS_key_bytes = 45 + TLS_key_path = 46 + TLS_verify_peers = 47 + Buggify_enable = 48 + Buggify_disable = 49 + Buggify_section_activated_probability = 50 + Buggify_section_fired_probability = 51 + TLS_ca_bytes = 52 + TLS_ca_path = 53 + TLS_password = 54 + disable_multi_version_client_api = 60 + callbacks_on_external_threads = 61 + external_client_library = 62 + external_client_directory = 63 + disable_local_client = 64 + client_threads_per_version = 65 + disable_client_statistics_logging = 70 + enable_slow_task_profiling = 71 + enable_run_loop_profiling = 71 + client_buggify_enable = 80 + client_buggify_disable = 81 + client_buggify_section_activated_probability = 82 + client_buggify_section_fired_probability = 83 + distributed_client_tracer = 90 + +class DatabaseOptions: + location_cache_size = 10 + max_watches = 20 + machine_id = 21 + datacenter_id = 22 + snapshot_ryw_enable = 26 + snapshot_ryw_disable = 27 + transaction_logging_max_field_length = 405 + transaction_timeout = 500 + transaction_retry_limit = 501 + transaction_max_retry_delay = 502 + transaction_size_limit = 503 + transaction_causal_read_risky = 504 + transaction_include_port_in_address = 505 + transaction_bypass_unreadable = 700 + use_config_database = 800 + test_causal_read_risky = 900 + + +class TransactionOptions: + causal_write_risky = 10 + causal_read_risky = 20 + causal_read_disable = 21 + include_port_in_address = 23 + next_write_no_write_conflict_range = 30 + read_your_writes_disable = 51 + read_ahead_disable = 52 + durability_datacenter = 110 + durability_risky = 120 + durability_dev_null_is_web_scale = 130 + priority_system_immediate = 200 + priority_batch = 201 + initialize_new_database = 300 + access_system_keys = 301 + read_system_keys = 302 + raw_access = 303 + debug_retry_logging = 401 + transaction_logging_enable = 402 + debug_transaction_identifier = 403 + log_transaction = 404 + transaction_logging_max_field_length = 405 + server_request_tracing = 406 + timeout = 500 + retry_limit = 501 + max_retry_delay = 502 + size_limit = 503 + snapshot_ryw_enable = 600 + snapshot_ryw_disable = 601 + lock_aware = 700 + used_during_commit_protection_disable = 701 + read_lock_aware = 702 + use_provisional_proxies = 711 + report_conflicting_keys = 712 + special_key_space_relaxed = 713 + special_key_space_enable_writes = 714 + tag = 800 + auto_throttle_tag = 801 + span_parent = 900 + expensive_clear_cost_estimation_enable = 1000 + bypass_unreadable = 1100 + use_grv_cache = 1101 diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py index d5e6e0da7f..87416e9414 100644 --- a/contrib/TestHarness2/test_harness/app.py +++ b/contrib/TestHarness2/test_harness/app.py @@ -1,7 +1,6 @@ import argparse import random -import test_harness.config from test_harness.config import config from test_harness.run import TestRunner @@ -9,7 +8,7 @@ from test_harness.run import TestRunner if __name__ == '__main__': # seed the random number generator parser = argparse.ArgumentParser('TestHarness') - test_harness.config.build_arguments(parser) + config.build_arguments(parser) test_runner = TestRunner() # initialize arguments parser.add_argument('--joshua-dir', type=str, help='Where to write FDB data to', required=False) @@ -18,7 +17,7 @@ if __name__ == '__main__': help='A base64 encoded list of statistics (used to reproduce runs)', required=False) args = parser.parse_args() - test_harness.config.extract_args(args) - random.seed(config.JOSHUA_SEED) + config.extract_args(args) + random.seed(config.joshua_seed) if not test_runner.run(args.stats): exit(1) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 4659833a3b..b65e68e169 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -1,12 +1,13 @@ from __future__ import annotations import argparse +import collections import copy import os import random from enum import Enum from pathlib import Path -from typing import List, Any +from typing import List, Any, OrderedDict class BuggifyOptionValue(Enum): @@ -18,11 +19,12 @@ class BuggifyOptionValue(Enum): class BuggifyOption: def __init__(self, val: str | None = None): self.value = BuggifyOptionValue.RANDOM - v = val.lower() - if v == 'on' or v == '1' or v == 'true': - self.value = BuggifyOptionValue.ON - elif v == 'off' or v == '0' or v == 'false': - self.value = BuggifyOptionValue.OFF + if val is not None: + v = val.lower() + if v == 'on' or v == '1' or v == 'true': + self.value = BuggifyOptionValue.ON + elif v == 'off' or v == '0' or v == 'false': + self.value = BuggifyOptionValue.OFF class ConfigValue: @@ -87,19 +89,72 @@ configuration: List[ConfigValue] = [ class Config: def __init__(self): - for val in configuration: - super().__setattr__(val.name.upper(), val.value) + self.kill_seconds: int = 30 * 60 + self.kill_seconds_args = {'help': 'Timeout for individual test'} + self.buggify_on_ratio: float = 0.8 + self.buggify_on_ratio_args = {'help': 'Probability that buggify is turned on'} + self.write_run_times = False + self.write_run_times_args = {'help': 'Write back probabilities after each test run', + 'action': 'store_true'} + self.unseed_check_ratio: float = 0.05 + self.unseed_check_ratio_args = {'help': 'Probability for doing determinism check'} + self.test_dirs: List[str] = ['slow', 'fast', 'restarting', 'rare', 'noSim'] + self.test_dirs_args: dict = {'nargs': '*'} + self.trace_format: str = 'json' + self.trace_format_args = {'choices': ['json', 'xml']} + self.crash_on_error: bool = True + self.crash_on_error_args = {'long_name': 'no_crash', 'action': 'store_false'} + self.max_warnings: int = 10 + self.max_warnings_args = {'short_name': 'W'} + self.max_errors: int = 10 + self.max_errors_args = {'short_name': 'E'} + self.old_binaries_path: Path = Path('/opt/joshua/global_data/oldBinaries') + self.old_binaries_path = {'help': 'Path to the directory containing the old fdb binaries'} + self.use_valgrind: bool = False + self.use_valgrind_args = {'action': 'store_true'} + self.buggify = BuggifyOption('random') + self.buggify_args = {'short_name': 'b', 'choices': ['on', 'off', 'random']} + self.pretty_print: bool = False + self.pretty_print_args = {'short_name': 'P', 'action': 'store_true'} + self.clean_up = True + self.clean_up_args = {'action': 'store_false'} + self.run_dir: Path = Path('tmp') + self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))) + self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'} + self.print_coverage = False + self.print_coverage_args = {'action': 'store_true'} + self.binary = Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver') + self.binary_args = {'help': 'Path to executable'} + self.output_format: str = 'xml' + self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']} + self.config_map = self._build_map() + + def _build_map(self): + config_map: OrderedDict[str, ConfigValue] = collections.OrderedDict() + for attr in dir(self): + obj = getattr(self, attr) + if attr.startswith('_') or callable(obj): + continue + if attr.endswith('_args'): + name = attr[0:-len('_args')] + assert name in config_map + print('assert isinstance({}, dict) type={}, obj={}'.format(attr, type(obj), obj)) + assert isinstance(obj, dict) + for k, v in obj.items(): + config_map[name].kwargs[k] = v + else: + kwargs = {'type': type(obj), 'default': obj} + config_map[attr] = ConfigValue(attr, **kwargs) + return config_map + + def build_arguments(self, parser: argparse.ArgumentParser): + for val in self.config_map.values(): + val.add_to_args(parser) + + def extract_args(self, args: argparse.Namespace): + for val in self.config_map.values(): + k, v = val.get_value(args) + config.__setattr__(k, v) config = Config() - - -def build_arguments(parser: argparse.ArgumentParser): - for val in configuration: - val.add_to_args(parser) - - -def extract_args(args: argparse.Namespace): - for val in configuration: - k, v = val.get_value(args) - config.__setattr__(k.upper(), v) diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index ac1de09989..35f65c3544 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -5,7 +5,7 @@ from typing import OrderedDict import fdb import struct -from run import StatFetcher, TestDescription +from test_harness.run import StatFetcher, TestDescription class FDBStatFetcher(StatFetcher): diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index c071edee55..209d5579c5 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -17,7 +17,7 @@ import uuid from test_harness import version from test_harness.config import config -from typing import List, Pattern, Callable, OrderedDict +from typing import List, Pattern, Callable, OrderedDict, Dict from pathlib import Path from test_harness.summarize import Summary, SummaryTree @@ -47,7 +47,7 @@ class FileStatsFetcher(StatFetcher): def __init__(self, stat_file: str, tests: OrderedDict[str, TestDescription]): super().__init__(tests) self.stat_file = stat_file - self.last_state = {} + self.last_state: Dict[str, int] = {} def read_stats(self): p = Path(self.stat_file) @@ -81,7 +81,7 @@ class TestPicker: self.follow_test: Pattern = re.compile(r".*-[2-9]\d*\.(txt|toml)") for subdir in self.test_dir.iterdir(): - if subdir.is_dir() and subdir.name in config.TEST_DIRS: + if subdir.is_dir() and subdir.name in config.test_dirs: self.walk_test_dir(subdir) self.fetcher = fetcher(self.tests) @@ -196,7 +196,7 @@ class TestPicker: class OldBinaries: def __init__(self): self.first_file_expr = re.compile(r'.*-1\.(txt|toml)') - self.old_binaries_path: Path = config.OLD_BINARIES_PATH + self.old_binaries_path: Path = config.old_binaries_path self.binaries: OrderedDict[version.Version, Path] = collections.OrderedDict() if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir(): return @@ -215,20 +215,20 @@ class OldBinaries: def choose_binary(self, test_file: Path) -> Path: if len(self.binaries) == 0: - return config.BINARY + return config.binary max_version = version.Version.max_version() min_version = version.Version.parse('5.0.0') dirs = test_file.parent.parts if 'restarting' not in dirs: - return config.BINARY + return config.binary version_expr = dirs[-1].split('_') first_file = self.first_file_expr.match(test_file.name) is not None if first_file and version_expr[0] == 'to': # downgrade test -- first binary should be current one - return config.BINARY + return config.binary if not first_file and version_expr[0] == 'from': # upgrade test -- we only return an old version for the first test file - return config.BINARY + return config.binary if version_expr[0] == 'from' or version_expr[0] == 'to': min_version = version.Version.parse(version_expr[1]) if len(version_expr) == 4 and version_expr[2] == 'until': @@ -286,11 +286,11 @@ class TestRun: self.stats: str | None = stats self.expected_unseed: int | None = expected_unseed self.err_out: str = 'error.xml' - self.use_valgrind: bool = config.USE_VALGRIND - self.old_binary_path: str = config.OLD_BINARIES_PATH - self.buggify_enabled: bool = random.random() < config.BUGGIFY_ON_RATIO + self.use_valgrind: bool = config.use_valgrind + self.old_binary_path: Path = config.old_binaries_path + self.buggify_enabled: bool = random.random() < config.buggify_on_ratio self.fault_injection_enabled: bool = True - self.trace_format = config.TRACE_FORMAT + self.trace_format = config.trace_format # state for the run self.retryable_error: bool = False self.summary: Summary | None = None @@ -326,9 +326,9 @@ class TestRun: command.append('--restarting') if self.buggify_enabled: command += ['-b', 'on'] - if config.CRASH_ON_ERROR: + if config.crash_on_error: command.append('--crash') - temp_path = config.RUN_DIR / str(self.uid) + temp_path = config.run_dir / str(self.uid) temp_path.mkdir(parents=True, exist_ok=True) # self.log_test_plan(out) @@ -337,7 +337,7 @@ class TestRun: process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=temp_path) did_kill = False try: - process.wait(20 * config.KILL_SECONDS if self.use_valgrind else config.KILL_SECONDS) + process.wait(20 * config.kill_seconds if self.use_valgrind else config.kill_seconds) except subprocess.TimeoutExpired: process.kill() did_kill = True @@ -363,12 +363,12 @@ class TestRunner: def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str): def fdb_fetcher(tests: OrderedDict[str, TestDescription]): from . import fdb - self.stat_fetcher = fdb.FDBStatFetcher(cluster_file, app_dir, tests) + self.stat_fetcher = lambda x: fdb.FDBStatFetcher(cluster_file, app_dir, x) self.stat_fetcher = fdb_fetcher def backup_sim_dir(self, seed: int): - temp_dir = config.RUN_DIR / str(self.uid) + temp_dir = config.run_dir / str(self.uid) src_dir = temp_dir / 'simfdb' assert src_dir.is_dir() dest_dir = temp_dir / 'simfdb.{}'.format(seed) @@ -376,7 +376,7 @@ class TestRunner: shutil.copytree(src_dir, dest_dir) def restore_sim_dir(self, seed: int): - temp_dir = config.RUN_DIR / str(self.uid) + temp_dir = config.run_dir / str(self.uid) src_dir = temp_dir / 'simfdb.{}'.format(seed) assert src_dir.exists() dest_dir = temp_dir / 'simfdb' @@ -387,7 +387,7 @@ class TestRunner: count = 0 for file in test_files: binary = self.binary_chooser.choose_binary(file) - unseed_check = random.random() < config.UNSEED_CHECK_RATIO + unseed_check = random.random() < config.unseed_check_ratio if unseed_check and count != 0: # for restarting tests we will need to restore the sim2 after the first run self.backup_sim_dir(seed + count - 1) @@ -395,14 +395,18 @@ class TestRunner: stats=test_picker.dump_stats()) success = run.run() test_picker.add_time(file, run.run_time) - if success and unseed_check and run.summary.unseed is not None: + assert not success or run.summary is not None + if success and unseed_check and run.summary is not None and run.summary.unseed is not None: self.restore_sim_dir(seed + count - 1) run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed) success = run2.run() - test_picker.add_time(file, run2.run_time) - run2.summary.out.dump(sys.stdout) - run.summary.out.dump(sys.stdout) + assert not success or run2.summary is not None + if success and run2.summary is not None: + test_picker.add_time(file, run2.run_time) + run2.summary.out.dump(sys.stdout) + if run.summary is not None: + run.summary.out.dump(sys.stdout) if not success: return False count += 1 @@ -410,7 +414,6 @@ class TestRunner: def run(self, stats: str | None) -> bool: seed = random.randint(0, 2 ** 32 - 1) - # unseed_check: bool = random.random() < Config.UNSEED_CHECK_RATIO test_picker = TestPicker(Path(self.test_path), self.stat_fetcher) if stats is not None: test_picker.load_stats(stats) @@ -418,6 +421,6 @@ class TestRunner: test_picker.fetch_stats() test_files = test_picker.choose_test() success = self.run_tests(test_files, seed, test_picker) - if config.CLEAN_UP: - shutil.rmtree(config.RUN_DIR / str(self.uid)) + if config.clean_up: + shutil.rmtree(config.run_dir / str(self.uid)) return success diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 3b5667434a..7274b7b1ad 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -12,7 +12,7 @@ import xml.sax.handler import xml.sax.saxutils from pathlib import Path -from typing import List, Dict, TextIO, Callable, Optional, OrderedDict +from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple from xml.dom import minidom from test_harness.config import config @@ -27,8 +27,8 @@ class SummaryTree: def append(self, element: SummaryTree): self.children.append(element) - def to_dict(self) -> Dict: - res = {'Type': self.name} + def to_dict(self) -> Dict[str, Any]: + res: Dict[str, Any] = {'Type': self.name} for k, v in self.attributes.items(): res[k] = v children = [] @@ -39,17 +39,7 @@ class SummaryTree: return res def to_json(self, out: TextIO): - json.dump(self.to_dict(), out, indent=(' ' if config.PRETTY_PRINT else None)) - - def append_xml_element(self, doc: minidom.Document, root=True, parent: minidom.Element | None = None): - assert root or parent is not None - this: minidom.Element = doc.createElement(self.name) - for k, v in self.attributes: - this.setAttribute(k, v) - for child in self.children: - child.append_xml_element(doc, root=False, parent=this) - if not root: - parent.appendChild(this) + json.dump(self.to_dict(), out, indent=(' ' if config.pretty_print else None)) def to_xml(self, out: TextIO, prefix: str = ''): # minidom doesn't support omitting the xml declaration which is a problem for joshua @@ -59,7 +49,7 @@ class SummaryTree: attrs.append('{}="{}"'.format(k, xml.sax.saxutils.escape(v))) elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' ')) out.write(elem) - if config.PRETTY_PRINT: + if config.pretty_print: curr_line_len = len(elem) for i in range(len(attrs)): attr_len = len(attrs[i]) @@ -80,28 +70,28 @@ class SummaryTree: else: out.write('>') for child in self.children: - if config.PRETTY_PRINT: + if config.pretty_print: out.write('\n') - child.to_xml(out, prefix=(' {}'.format(prefix) if config.PRETTY_PRINT else prefix)) + child.to_xml(out, prefix=(' {}'.format(prefix) if config.pretty_print else prefix)) if len(self.children) > 0: - out.write('{}'.format(('\n' if config.PRETTY_PRINT else ''), self.name)) + out.write('{}'.format(('\n' if config.pretty_print else ''), self.name)) def dump(self, out: TextIO): - if config.OUTPUT_FORMAT == 'json': + if config.output_format == 'json': self.to_json(out) else: self.to_xml(out) -ParserCallback = Callable[[Optional[Dict[str, Optional[str]]]], Optional[str]] +ParserCallback = Callable[[Dict[str, str]], Optional[str]] class ParseHandler: def __init__(self, out: SummaryTree): self.out = out - self.events: OrderedDict[(str, str), List[ParserCallback]] = collections.OrderedDict() + self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict() - def add_handler(self, attr: (str, str), callback: ParserCallback) -> None: + def add_handler(self, attr: Tuple[str, str], callback: ParserCallback) -> None: if attr in self.events: self.events[attr].append(callback) else: @@ -155,6 +145,7 @@ class XmlParser(Parser, xml.sax.handler.ContentHandler): attributes: Dict[str, str] = {} for name in attrs.getNames(): attributes[name] = attrs.getValue(name) + assert self.handler is not None self.handler.handle(attributes) @@ -266,32 +257,48 @@ class Coverage: self.line = int(line) self.comment = comment - def __eq__(self, other: Coverage) -> bool: - return self.file == other.file and self.line == other.line and self.comment == other.comment + def to_tuple(self) -> Tuple[str, int, str | None]: + return self.file, self.line, self.comment - def __lt__(self, other: Coverage) -> bool: - if self.file != other.file: - return self.file < other.file - elif self.line != other.line: - return self.line < other.line - elif self.comment != other.comment: - if self.comment is None: - return True - elif other.comment is None: - return False - else: - return self.comment < other.comment + def __eq__(self, other) -> bool: + if isinstance(other, tuple) and len(other) == 3: + return self.to_tuple() == other + elif isinstance(other, Coverage): + return self.to_tuple() == other.to_tuple() else: return False - def __le__(self, other: Coverage) -> bool: - return self.__eq__(other) or self.__lt__(other) + def __lt__(self, other) -> bool: + if isinstance(other, tuple) and len(other) == 3: + return self.to_tuple() < other + elif isinstance(other, Coverage): + return self.to_tuple() < other.to_tuple() + else: + return False + + def __le__(self, other) -> bool: + if isinstance(other, tuple) and len(other) == 3: + return self.to_tuple() <= other + elif isinstance(other, Coverage): + return self.to_tuple() <= other.to_tuple() + else: + return False def __gt__(self, other: Coverage) -> bool: - return not self.__le__(other) + if isinstance(other, tuple) and len(other) == 3: + return self.to_tuple() > other + elif isinstance(other, Coverage): + return self.to_tuple() > other.to_tuple() + else: + return False def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) + if isinstance(other, tuple) and len(other) == 3: + return self.to_tuple() >= other + elif isinstance(other, Coverage): + return self.to_tuple() >= other.to_tuple() + else: + return False def __hash__(self): return hash((self.file, self.line, self.comment)) @@ -350,7 +357,7 @@ class Summary: self.out.attributes['TestUID'] = str(uid) if stats is not None: self.out.attributes['Statistics'] = stats - self.out.attributes['JoshuaSeed'] = str(config.JOSHUA_SEED) + self.out.attributes['JoshuaSeed'] = str(config.joshua_seed) self.handler = ParseHandler(self.out) self.register_handlers() @@ -372,7 +379,7 @@ class Summary: and self.test_end_found def done(self): - if config.PRINT_COVERAGE: + if config.print_coverage: for k, v in self.coverage.items(): child = SummaryTree('CodeCoverage') child.attributes['File'] = k.file @@ -382,12 +389,12 @@ class Summary: if k.comment is not None and len(k.comment): child.attributes['Comment'] = k.comment self.out.append(child) - if self.warnings > config.MAX_WARNINGS: + if self.warnings > config.max_warnings: child = SummaryTree('WarningLimitExceeded') child.attributes['Severity'] = '30' child.attributes['WarningCount'] = str(self.warnings) self.out.append(child) - if self.errors > config.MAX_ERRORS: + if self.errors > config.max_errors: child = SummaryTree('ErrorLimitExceeded') child.attributes['Severity'] = '40' child.attributes['ErrorCount'] = str(self.warnings) @@ -423,6 +430,7 @@ class Summary: self.out.attributes['Ok'] = '1' if self.ok() else '0' def parse_file(self, file: Path): + parser: Parser if file.suffix == '.json': parser = JsonParser() elif file.suffix == '.xml': @@ -430,7 +438,9 @@ class Summary: else: child = SummaryTree('TestHarnessBug') child.attributes['File'] = __file__ - child.attributes['Line'] = str(inspect.getframeinfo(inspect.currentframe()).lineno) + frame = inspect.currentframe() + if frame is not None: + child.attributes['Line'] = str(inspect.getframeinfo(frame).lineno) child.attributes['Details'] = 'Unexpected suffix {} for file {}'.format(file.suffix, file.name) self.error = True self.out.append(child) @@ -509,7 +519,7 @@ class Summary: def parse_warning(attrs: Dict[str, str]): self.warnings += 1 - if self.warnings > config.MAX_WARNINGS: + if self.warnings > config.max_warnings: return child = SummaryTree(attrs['Type']) for k, v in attrs.items(): @@ -522,7 +532,7 @@ class Summary: def parse_error(attrs: Dict[str, str]): self.errors += 1 self.error = True - if self.errors > config.MAX_ERRORS: + if self.errors > config.max_errors: return child = SummaryTree(attrs['Type']) for k, v in attrs.items(): From 0971ee5f7a0e55374ddde4b47335395cdd45adfe Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 16 Aug 2022 11:31:37 -0600 Subject: [PATCH 03/29] enable new test harness for correctness builds --- cmake/AddFdbTest.cmake | 29 ++++++++-- contrib/CMakeLists.txt | 1 + contrib/Joshua/scripts/correctnessTest.sh | 4 +- contrib/TestHarness2/CMakeLists.txt | 0 contrib/TestHarness2/test_harness/app.py | 40 ++++++++----- contrib/TestHarness2/test_harness/config.py | 58 ++++++++----------- contrib/TestHarness2/test_harness/run.py | 32 +++++----- .../TestHarness2/test_harness/summarize.py | 4 +- 8 files changed, 98 insertions(+), 70 deletions(-) create mode 100644 contrib/TestHarness2/CMakeLists.txt diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 78de24355d..13fa8d6a23 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -198,16 +198,17 @@ function(stage_correctness_package) set(src_dir "${src_dir}/") string(SUBSTRING ${src_dir} ${dir_len} -1 dest_dir) string(SUBSTRING ${file} ${dir_len} -1 rel_out_file) - set(out_file ${STAGE_OUT_DIR}/${rel_out_file}) + set(out_file ${STAGE_OUT_DIR}/${rel_out_file}) list(APPEND external_files ${out_file}) - add_custom_command( + add_custom_command( OUTPUT ${out_file} - DEPENDS ${file} - COMMAND ${CMAKE_COMMAND} -E copy ${file} ${out_file} - COMMENT "Copying ${STAGE_CONTEXT} external file ${file}" - ) + DEPENDS ${file} + COMMAND ${CMAKE_COMMAND} -E copy ${file} ${out_file} + COMMENT "Copying ${STAGE_CONTEXT} external file ${file}" + ) endforeach() endforeach() + list(APPEND package_files ${STAGE_OUT_DIR}/bin/fdbserver ${STAGE_OUT_DIR}/bin/coverage.fdbserver.xml ${STAGE_OUT_DIR}/bin/coverage.fdbclient.xml @@ -217,6 +218,7 @@ function(stage_correctness_package) ${STAGE_OUT_DIR}/bin/TraceLogHelper.dll ${STAGE_OUT_DIR}/CMakeCache.txt ) + add_custom_command( OUTPUT ${package_files} DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt @@ -227,6 +229,7 @@ function(stage_correctness_package) ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll + ${harness_files} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/CMakeCache.txt ${STAGE_OUT_DIR} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbserver ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml @@ -238,6 +241,20 @@ function(stage_correctness_package) ${STAGE_OUT_DIR}/bin COMMENT "Copying files for ${STAGE_CONTEXT} package" ) + + set(test_harness_dir "${CMAKE_SOURCE_DIR}/contrib/TestHarness2") + file(GLOB_RECURSE test_harness2_files RELATIVE "${test_harness_dir}" CONFIGURE_DEPENDS "${test_harness_dir}/*.py") + foreach(file IN LISTS test_harness2_files) + set(src_file "${test_harness_dir}/${file}") + set(out_file "${STAGE_OUT_DIR}/${file}") + get_filename_component(dir "${out_file}" DIRECTORY) + file(MAKE_DIRECTORY "${dir}") + add_custom_command(OUTPUT ${out_file} + COMMAND ${CMAKE_COMMAND} -E copy "${src_file}" "${out_file}" + DEPENDS "${src_file}") + list(APPEND package_files "${out_file}") + endforeach() + list(APPEND package_files ${test_files} ${external_files}) if(STAGE_OUT_FILES) set(${STAGE_OUT_FILES} ${package_files} PARENT_SCOPE) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 75ca06243f..ad741de86d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -18,3 +18,4 @@ if(NOT WIN32) add_subdirectory(TestHarness) endif() add_subdirectory(mockkms) +add_subdirectory(TestHarness2) diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh index a617d81088..bee09acf25 100755 --- a/contrib/Joshua/scripts/correctnessTest.sh +++ b/contrib/Joshua/scripts/correctnessTest.sh @@ -4,4 +4,6 @@ export ASAN_OPTIONS="detect_leaks=0" OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}" -mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false +#mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false + +python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR} diff --git a/contrib/TestHarness2/CMakeLists.txt b/contrib/TestHarness2/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py index 87416e9414..17ddb84474 100644 --- a/contrib/TestHarness2/test_harness/app.py +++ b/contrib/TestHarness2/test_harness/app.py @@ -1,23 +1,35 @@ import argparse import random +import sys +import traceback from test_harness.config import config from test_harness.run import TestRunner +from test_harness.summarize import SummaryTree if __name__ == '__main__': - # seed the random number generator - parser = argparse.ArgumentParser('TestHarness') - config.build_arguments(parser) - test_runner = TestRunner() - # initialize arguments - parser.add_argument('--joshua-dir', type=str, help='Where to write FDB data to', required=False) - parser.add_argument('-C', '--cluster-file', type=str, help='Path to fdb cluster file', required=False) - parser.add_argument('--stats', type=str, - help='A base64 encoded list of statistics (used to reproduce runs)', - required=False) - args = parser.parse_args() - config.extract_args(args) - random.seed(config.joshua_seed) - if not test_runner.run(args.stats): + try: + # seed the random number generator + parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + config.build_arguments(parser) + # initialize arguments + parser.add_argument('--joshua-dir', type=str, help='Where to write FDB data to', required=False) + parser.add_argument('-C', '--cluster-file', type=str, help='Path to fdb cluster file', required=False) + parser.add_argument('--stats', type=str, + help='A base64 encoded list of statistics (used to reproduce runs)', + required=False) + args = parser.parse_args() + config.extract_args(args) + random.seed(config.joshua_seed) + test_runner = TestRunner() + if not test_runner.run(args.stats): + exit(1) + except Exception as e: + _, _, exc_traceback = sys.exc_info() + error = SummaryTree('TestHarnessError') + error.attributes['Severity'] = '40' + error.attributes['ErrorMessage'] = str(e) + error.attributes['Trace'] = repr(traceback.format_tb(exc_traceback)) + error.dump(sys.stdout) exit(1) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index b65e68e169..e82bc5c034 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -53,40 +53,21 @@ class ConfigValue: del kwargs['short_name'] long_name = long_name.replace('_', '-') if short_name is None: + # line below is useful for debugging + # print('add_argument(\'--{}\', [{{{}}}])'.format(long_name, ', '.join(['\'{}\': \'{}\''.format(k, v) + # for k, v in kwargs.items()]))) parser.add_argument('--{}'.format(long_name), **kwargs) else: + # line below is useful for debugging + # print('add_argument(\'-{}\', \'--{}\', [{{{}}}])'.format(short_name, long_name, + # ', '.join(['\'{}\': \'{}\''.format(k, v) + # for k, v in kwargs.items()]))) parser.add_argument('-{}'.format(short_name), '--{}'.format(long_name), **kwargs) def get_value(self, args: argparse.Namespace) -> tuple[str, Any]: return self.name, args.__getattribute__(self.get_arg_name()) -configuration: List[ConfigValue] = [ - ConfigValue('kill_seconds', default=60 * 30, help='Timeout for individual test', type=int), - ConfigValue('buggify_on_ratio', default=0.8, help='Probability that buggify is turned on', type=float), - ConfigValue('write_run_times', default=False, help='Write back probabilities after each test run', - action='store_true'), - ConfigValue('unseed_check_ratio', default=0.05, help='Probability for doing determinism check', type=float), - ConfigValue('test_dirs', default=['slow', 'fast', 'restarting', 'rare', 'noSim'], nargs='*'), - ConfigValue('trace_format', default='json', choices=['json', 'xml']), - ConfigValue('crash_on_error', long_name='no_crash', default=True, action='store_false'), - ConfigValue('max_warnings', default=10, short_name='W', type=int), - ConfigValue('max_errors', default=10, short_name='E', type=int), - ConfigValue('old_binaries_path', default=Path('/opt/joshua/global_data/oldBinaries'), type=Path), - ConfigValue('use_valgrind', default=False, action='store_true'), - ConfigValue('buggify', short_name='b', default=BuggifyOption('random'), type=BuggifyOption, - choices=['on', 'off', 'random']), - ConfigValue('pretty_print', short_name='P', default=False, action='store_true'), - ConfigValue('clean_up', default=True), - ConfigValue('run_dir', default=Path('tmp'), type=Path), - ConfigValue('joshua_seed', default=int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))), type=int), - ConfigValue('print_coverage', default=False, action='store_true'), - ConfigValue('binary', default=Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver'), - help='Path to executable', type=Path), - ConfigValue('output_format', short_name='O', type=str, choices=['json', 'xml'], default='xml'), -] - - class Config: def __init__(self): self.kill_seconds: int = 30 * 60 @@ -99,17 +80,18 @@ class Config: self.unseed_check_ratio: float = 0.05 self.unseed_check_ratio_args = {'help': 'Probability for doing determinism check'} self.test_dirs: List[str] = ['slow', 'fast', 'restarting', 'rare', 'noSim'] - self.test_dirs_args: dict = {'nargs': '*'} + self.test_dirs_args: dict = {'nargs': '*', 'help': 'test_directories to look for files in'} self.trace_format: str = 'json' - self.trace_format_args = {'choices': ['json', 'xml']} + self.trace_format_args = {'choices': ['json', 'xml'], 'help': 'What format fdb should produce'} self.crash_on_error: bool = True - self.crash_on_error_args = {'long_name': 'no_crash', 'action': 'store_false'} + self.crash_on_error_args = {'long_name': 'no_crash', 'action': 'store_false', + 'help': 'Don\'t crash on first error'} self.max_warnings: int = 10 self.max_warnings_args = {'short_name': 'W'} self.max_errors: int = 10 self.max_errors_args = {'short_name': 'E'} self.old_binaries_path: Path = Path('/opt/joshua/global_data/oldBinaries') - self.old_binaries_path = {'help': 'Path to the directory containing the old fdb binaries'} + self.old_binaries_path_args = {'help': 'Path to the directory containing the old fdb binaries'} self.use_valgrind: bool = False self.use_valgrind_args = {'action': 'store_true'} self.buggify = BuggifyOption('random') @@ -117,7 +99,7 @@ class Config: self.pretty_print: bool = False self.pretty_print_args = {'short_name': 'P', 'action': 'store_true'} self.clean_up = True - self.clean_up_args = {'action': 'store_false'} + self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'} self.run_dir: Path = Path('tmp') self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))) self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'} @@ -138,12 +120,22 @@ class Config: if attr.endswith('_args'): name = attr[0:-len('_args')] assert name in config_map - print('assert isinstance({}, dict) type={}, obj={}'.format(attr, type(obj), obj)) assert isinstance(obj, dict) for k, v in obj.items(): + if k == 'action' and v in ['store_true', 'store_false']: + # we can't combine type and certain actions + del config_map[name].kwargs['type'] config_map[name].kwargs[k] = v else: - kwargs = {'type': type(obj), 'default': obj} + val_type = type(obj) + env_name = 'TH_{}'.format(attr.upper()) + new_val = os.getenv(env_name) + try: + if new_val is not None: + obj = val_type(new_val) + except: + pass + kwargs = {'type': val_type, 'default': obj} config_map[attr] = ConfigValue(attr, **kwargs) return config_map diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 209d5579c5..99d1cce05e 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -237,6 +237,8 @@ class OldBinaries: for ver, binary in self.binaries.items(): if min_version <= ver <= max_version: candidates.append(binary) + if len(candidates) == 0: + return config.binary return random.choice(candidates) @@ -293,12 +295,13 @@ class TestRun: self.trace_format = config.trace_format # state for the run self.retryable_error: bool = False - self.summary: Summary | None = None + self.summary: Summary = Summary(binary) self.run_time: int = 0 + self.success = self.run() def log_test_plan(self, out: SummaryTree): test_plan: SummaryTree = SummaryTree('TestPlan') - test_plan.attributes['TestUID'] = str(uuid) + test_plan.attributes['TestUID'] = str(self.uid) test_plan.attributes['RandomSeed'] = str(self.random_seed) test_plan.attributes['TestFile'] = str(self.test_file) test_plan.attributes['Buggify'] = '1' if self.buggify_enabled else '0' @@ -344,6 +347,9 @@ class TestRun: resources.stop() resources.join() self.run_time = round(resources.time()) + self.summary.runtime = resources.time() + self.summary.max_rss = resources.max_rss + self.summary.was_killed = did_kill self.summary = Summary(self.binary, temp_path, runtime=resources.time(), max_rss=resources.max_rss, was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file, expected_unseed=self.expected_unseed) @@ -363,7 +369,7 @@ class TestRunner: def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str): def fdb_fetcher(tests: OrderedDict[str, TestDescription]): from . import fdb - self.stat_fetcher = lambda x: fdb.FDBStatFetcher(cluster_file, app_dir, x) + return fdb.FDBStatFetcher(cluster_file, app_dir, tests) self.stat_fetcher = fdb_fetcher @@ -385,6 +391,7 @@ class TestRunner: def run_tests(self, test_files: List[Path], seed: int, test_picker: TestPicker) -> bool: count = 0 + result: bool = True for file in test_files: binary = self.binary_chooser.choose_binary(file) unseed_check = random.random() < config.unseed_check_ratio @@ -393,24 +400,19 @@ class TestRunner: self.backup_sim_dir(seed + count - 1) run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats()) - success = run.run() + run.summary.out.dump(sys.stdout) + result = result and run.success test_picker.add_time(file, run.run_time) - assert not success or run.summary is not None - if success and unseed_check and run.summary is not None and run.summary.unseed is not None: + if run.success and unseed_check and run.summary.unseed is not None: self.restore_sim_dir(seed + count - 1) run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed) - success = run2.run() - assert not success or run2.summary is not None - if success and run2.summary is not None: - test_picker.add_time(file, run2.run_time) - run2.summary.out.dump(sys.stdout) - if run.summary is not None: - run.summary.out.dump(sys.stdout) - if not success: + test_picker.add_time(file, run2.run_time) + run2.summary.out.dump(sys.stdout) + if not result: return False count += 1 - return True + return result def run(self, stats: str | None) -> bool: seed = random.randint(0, 2 ** 32 - 1) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 7274b7b1ad..0ead14d090 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -81,6 +81,7 @@ class SummaryTree: self.to_json(out) else: self.to_xml(out) + out.write('\n') ParserCallback = Callable[[Dict[str, str]], Optional[str]] @@ -330,7 +331,7 @@ class TraceFiles: class Summary: - def __init__(self, binary: Path, trace_dir: Path, runtime: float = 0, max_rss: int | None = None, + def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None, was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None, exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None): self.binary = binary @@ -362,6 +363,7 @@ class Summary: self.handler = ParseHandler(self.out) self.register_handlers() + def summarize(self, trace_dir: Path): trace_files = TraceFiles(trace_dir) if len(trace_files) == 0: self.error = True From 2704891c1a63f9b4ac48dfb9c0e695670a3d2396 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 16 Aug 2022 13:21:58 -0600 Subject: [PATCH 04/29] handle include and exclude, fix reporting --- contrib/TestHarness2/test_harness/config.py | 4 ++++ contrib/TestHarness2/test_harness/run.py | 22 +++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index e82bc5c034..c17b4eda2b 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -109,6 +109,10 @@ class Config: self.binary_args = {'help': 'Path to executable'} self.output_format: str = 'xml' self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']} + self.include_test_files: str = r'.*' + self.exclude_test_files: str = r'.^' + self.include_test_names: str = r'.*' + self.exclude_test_names: str = r'.^' self.config_map = self._build_map() def _build_map(self): diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 99d1cce05e..710dbb07d2 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -75,6 +75,10 @@ class TestPicker: def __init__(self, test_dir: Path, fetcher: StatFetcherCreator): if not test_dir.exists(): raise RuntimeError('{} is neither a directory nor a file'.format(test_dir)) + self.include_files_regex = re.compile(config.include_test_files) + self.exclude_files_regex = re.compile(config.exclude_test_files) + self.include_tests_regex = re.compile(config.include_test_names) + self.exclude_tests_regex = re.compile(config.exclude_test_names) self.test_dir: Path = test_dir self.tests: OrderedDict[str, TestDescription] = collections.OrderedDict() self.restart_test: Pattern = re.compile(r".*-\d+\.(txt|toml)") @@ -117,6 +121,8 @@ class TestPicker: idx += 1 def parse_txt(self, path: Path): + if self.include_files_regex.match(str(path)) is None or self.exclude_files_regex.match(str(path)) is not None: + return with path.open('r') as f: test_name: str | None = None test_class: str | None = None @@ -145,6 +151,9 @@ class TestPicker: test_class = test_name if priority is None: priority = 1.0 + if self.include_tests_regex.match(test_class) is None \ + or self.exclude_tests_regex.match(test_class) is not None: + return if test_class not in self.tests: self.tests[test_class] = TestDescription(path, test_class, priority) else: @@ -293,6 +302,7 @@ class TestRun: self.buggify_enabled: bool = random.random() < config.buggify_on_ratio self.fault_injection_enabled: bool = True self.trace_format = config.trace_format + self.temp_path = config.run_dir / str(self.uid) # state for the run self.retryable_error: bool = False self.summary: Summary = Summary(binary) @@ -331,13 +341,13 @@ class TestRun: command += ['-b', 'on'] if config.crash_on_error: command.append('--crash') - temp_path = config.run_dir / str(self.uid) - temp_path.mkdir(parents=True, exist_ok=True) + + self.temp_path.mkdir(parents=True, exist_ok=True) # self.log_test_plan(out) resources = ResourceMonitor() resources.start() - process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=temp_path) + process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.temp_path) did_kill = False try: process.wait(20 * config.kill_seconds if self.use_valgrind else config.kill_seconds) @@ -350,9 +360,11 @@ class TestRun: self.summary.runtime = resources.time() self.summary.max_rss = resources.max_rss self.summary.was_killed = did_kill - self.summary = Summary(self.binary, temp_path, runtime=resources.time(), max_rss=resources.max_rss, + self.summary = Summary(self.binary, runtime=resources.time(), max_rss=resources.max_rss, was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file, expected_unseed=self.expected_unseed) + self.summary.summarize(self.temp_path) + self.summary.out.dump(sys.stdout) return self.summary.ok() @@ -400,7 +412,6 @@ class TestRunner: self.backup_sim_dir(seed + count - 1) run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats()) - run.summary.out.dump(sys.stdout) result = result and run.success test_picker.add_time(file, run.run_time) if run.success and unseed_check and run.summary.unseed is not None: @@ -408,7 +419,6 @@ class TestRunner: run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed) test_picker.add_time(file, run2.run_time) - run2.summary.out.dump(sys.stdout) if not result: return False count += 1 From 2bf6a838b89fb7f7be107545d87596807a2025de Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Wed, 17 Aug 2022 16:27:44 -0600 Subject: [PATCH 05/29] basic functionality working --- contrib/TestHarness2/test_harness/app.py | 3 - contrib/TestHarness2/test_harness/config.py | 11 ++ contrib/TestHarness2/test_harness/run.py | 137 ++++++++++-------- .../TestHarness2/test_harness/summarize.py | 57 +++++++- contrib/TestHarness2/test_harness/version.py | 47 ++++-- 5 files changed, 172 insertions(+), 83 deletions(-) diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py index 17ddb84474..27edc0f065 100644 --- a/contrib/TestHarness2/test_harness/app.py +++ b/contrib/TestHarness2/test_harness/app.py @@ -1,5 +1,4 @@ import argparse -import random import sys import traceback @@ -10,7 +9,6 @@ from test_harness.summarize import SummaryTree if __name__ == '__main__': try: - # seed the random number generator parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.build_arguments(parser) # initialize arguments @@ -21,7 +19,6 @@ if __name__ == '__main__': required=False) args = parser.parse_args() config.extract_args(args) - random.seed(config.joshua_seed) test_runner = TestRunner() if not test_runner.run(args.stats): exit(1) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index c17b4eda2b..ddaf24a077 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -70,6 +70,7 @@ class ConfigValue: class Config: def __init__(self): + self.random = random.Random() self.kill_seconds: int = 30 * 60 self.kill_seconds_args = {'help': 'Timeout for individual test'} self.buggify_on_ratio: float = 0.8 @@ -102,6 +103,7 @@ class Config: self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'} self.run_dir: Path = Path('tmp') self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))) + self.random.seed(self.joshua_seed, version=2) self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'} self.print_coverage = False self.print_coverage_args = {'action': 'store_true'} @@ -110,9 +112,16 @@ class Config: self.output_format: str = 'xml' self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']} self.include_test_files: str = r'.*' + self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'} self.exclude_test_files: str = r'.^' + self.exclude_test_files_args = {'help': 'Don\'t consider test files whose path match against the given regex'} self.include_test_names: str = r'.*' + self.include_test_names_args = {'help': 'Only consider tests whose names match against the given regex'} self.exclude_test_names: str = r'.^' + self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'} + self.max_stderr_bytes: int = 1000 + self.write_stats: bool = True + self.read_stats: bool = True self.config_map = self._build_map() def _build_map(self): @@ -151,6 +160,8 @@ class Config: for val in self.config_map.values(): k, v = val.get_value(args) config.__setattr__(k, v) + if k == 'joshua_seed': + self.random.seed(self.joshua_seed, version=2) config = Config() diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 710dbb07d2..fc280052ad 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -8,21 +8,22 @@ import os import resource import shutil import subprocess -import random import re import sys import threading import time import uuid -from test_harness import version -from test_harness.config import config -from typing import List, Pattern, Callable, OrderedDict, Dict +from functools import total_ordering from pathlib import Path +from test_harness.version import Version +from test_harness.config import config +from typing import List, Pattern, OrderedDict, Dict from test_harness.summarize import Summary, SummaryTree +@total_ordering class TestDescription: def __init__(self, path: Path, name: str, priority: float): self.paths: List[Path] = [path] @@ -31,6 +32,18 @@ class TestDescription: # we only measure in seconds. Otherwise, keeping determinism will be difficult self.total_runtime: int = 0 + def __lt__(self, other): + if isinstance(other, TestDescription): + return self.name < other.name + else: + return self.name < str(other) + + def __eq__(self, other): + if isinstance(other, TestDescription): + return self.name < other.name + else: + return self.name < str(other.name) + class StatFetcher: def __init__(self, tests: OrderedDict[str, TestDescription]): @@ -68,11 +81,8 @@ class FileStatsFetcher(StatFetcher): json.dump(self.last_state, f) -StatFetcherCreator = Callable[[OrderedDict[str, TestDescription]], StatFetcher] - - class TestPicker: - def __init__(self, test_dir: Path, fetcher: StatFetcherCreator): + def __init__(self, test_dir: Path): if not test_dir.exists(): raise RuntimeError('{} is neither a directory nor a file'.format(test_dir)) self.include_files_regex = re.compile(config.include_test_files) @@ -87,7 +97,8 @@ class TestPicker: for subdir in self.test_dir.iterdir(): if subdir.is_dir() and subdir.name in config.test_dirs: self.walk_test_dir(subdir) - self.fetcher = fetcher(self.tests) + self.fetcher: StatFetcher + # self.fetcher = fetcher(self.tests) def add_time(self, test_file: Path, run_time: int) -> None: # getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother @@ -99,8 +110,8 @@ class TestPicker: break if test_name is not None: break - assert test_name is not None - self.fetcher.add_run_time(test_name, run_time) + # assert test_name is not None + # self.fetcher.add_run_time(test_name, run_time) def dump_stats(self) -> str: res = array.array('I') @@ -109,7 +120,8 @@ class TestPicker: return base64.standard_b64encode(res.tobytes()).decode('utf-8') def fetch_stats(self): - self.fetcher.read_stats() + # self.fetcher.read_stats() + pass def load_stats(self, serialized: str): times = array.array('I') @@ -121,7 +133,7 @@ class TestPicker: idx += 1 def parse_txt(self, path: Path): - if self.include_files_regex.match(str(path)) is None or self.exclude_files_regex.match(str(path)) is not None: + if self.include_files_regex.search(str(path)) is None or self.exclude_files_regex.search(str(path)) is not None: return with path.open('r') as f: test_name: str | None = None @@ -151,8 +163,8 @@ class TestPicker: test_class = test_name if priority is None: priority = 1.0 - if self.include_tests_regex.match(test_class) is None \ - or self.exclude_tests_regex.match(test_class) is not None: + if self.include_tests_regex.search(test_class) is None \ + or self.exclude_tests_regex.search(test_class) is not None: return if test_class not in self.tests: self.tests[test_class] = TestDescription(path, test_class, priority) @@ -193,9 +205,10 @@ class TestPicker: candidates = [v] elif this_time == min_runtime: candidates.append(v) - choice = random.randint(0, len(candidates) - 1) + candidates.sort() + choice = config.random.randint(0, len(candidates) - 1) test = candidates[choice] - result = test.paths[random.randint(0, len(test.paths) - 1)] + result = test.paths[config.random.randint(0, len(test.paths) - 1)] if self.restart_test.match(result.name): return self.list_restart_files(result) else: @@ -206,7 +219,7 @@ class OldBinaries: def __init__(self): self.first_file_expr = re.compile(r'.*-1\.(txt|toml)') self.old_binaries_path: Path = config.old_binaries_path - self.binaries: OrderedDict[version.Version, Path] = collections.OrderedDict() + self.binaries: OrderedDict[Version, Path] = collections.OrderedDict() if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir(): return exec_pattern = re.compile(r'fdbserver-\d+\.\d+\.\d+(\.exe)?') @@ -220,13 +233,14 @@ class OldBinaries: version_str = file.name.split('-')[1] if version_str.endswith('.exe'): version_str = version_str[0:-len('.exe')] - self.binaries[version.Version.parse(version_str)] = file + ver = Version.parse(version_str) + self.binaries[ver] = file def choose_binary(self, test_file: Path) -> Path: if len(self.binaries) == 0: return config.binary - max_version = version.Version.max_version() - min_version = version.Version.parse('5.0.0') + max_version = Version.max_version() + min_version = Version.parse('5.0.0') dirs = test_file.parent.parts if 'restarting' not in dirs: return config.binary @@ -239,16 +253,16 @@ class OldBinaries: # upgrade test -- we only return an old version for the first test file return config.binary if version_expr[0] == 'from' or version_expr[0] == 'to': - min_version = version.Version.parse(version_expr[1]) + min_version = Version.parse(version_expr[1]) if len(version_expr) == 4 and version_expr[2] == 'until': - max_version = version.Version.parse(version_expr[3]) + max_version = Version.parse(version_expr[3]) candidates: List[Path] = [] for ver, binary in self.binaries.items(): if min_version <= ver <= max_version: candidates.append(binary) if len(candidates) == 0: return config.binary - return random.choice(candidates) + return config.random.choice(candidates) def is_restarting_test(test_file: Path): @@ -287,7 +301,7 @@ class ResourceMonitor(threading.Thread): class TestRun: def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID, restarting: bool = False, test_determinism: bool = False, - stats: str | None = None, expected_unseed: int | None = None): + stats: str | None = None, expected_unseed: int | None = None, will_restart: bool = False): self.binary = binary self.test_file = test_file self.random_seed = random_seed @@ -299,13 +313,16 @@ class TestRun: self.err_out: str = 'error.xml' self.use_valgrind: bool = config.use_valgrind self.old_binary_path: Path = config.old_binaries_path - self.buggify_enabled: bool = random.random() < config.buggify_on_ratio + self.buggify_enabled: bool = config.random.random() < config.buggify_on_ratio self.fault_injection_enabled: bool = True self.trace_format = config.trace_format + if Version.of_binary(self.binary) < "6.1.0": + self.trace_format = None self.temp_path = config.run_dir / str(self.uid) # state for the run self.retryable_error: bool = False - self.summary: Summary = Summary(binary) + self.summary: Summary = Summary(binary, uid=self.uid, stats=self.stats, expected_unseed=self.expected_unseed, + will_restart=will_restart) self.run_time: int = 0 self.success = self.run() @@ -332,9 +349,11 @@ class TestRun: command += [str(self.binary.absolute()), '-r', 'test' if is_no_sim(self.test_file) else 'simulation', '-f', str(self.test_file), - '-s', str(self.random_seed), - '-fi', 'on' if self.fault_injection_enabled else 'off', - '--trace_format', self.trace_format] + '-s', str(self.random_seed)] + if self.trace_format is not None: + command += ['--trace_format', self.trace_format] + if Version.of_binary(self.binary) >= '7.1.0': + command += ['-fi', 'on' if self.fault_injection_enabled else 'off'] if self.restarting: command.append('--restarting') if self.buggify_enabled: @@ -347,12 +366,16 @@ class TestRun: # self.log_test_plan(out) resources = ResourceMonitor() resources.start() - process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.temp_path) + process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=self.temp_path, + text=True) did_kill = False + timeout = 20 * config.kill_seconds if self.use_valgrind else config.kill_seconds + err_out: str try: - process.wait(20 * config.kill_seconds if self.use_valgrind else config.kill_seconds) + _, err_out = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() + _, err_out = process.communicate() did_kill = True resources.stop() resources.join() @@ -360,10 +383,9 @@ class TestRun: self.summary.runtime = resources.time() self.summary.max_rss = resources.max_rss self.summary.was_killed = did_kill - self.summary = Summary(self.binary, runtime=resources.time(), max_rss=resources.max_rss, - was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file, - expected_unseed=self.expected_unseed) - self.summary.summarize(self.temp_path) + self.summary.valgrind_out_file = valgrind_file + self.summary.error_out = err_out + self.summary.summarize(self.temp_path, ' '.join(command)) self.summary.out.dump(sys.stdout) return self.summary.ok() @@ -371,19 +393,11 @@ class TestRun: class TestRunner: def __init__(self): self.uid = uuid.uuid4() - self.test_path: str = 'tests' + self.test_path: Path = Path('tests') self.cluster_file: str | None = None self.fdb_app_dir: str | None = None - self.stat_fetcher: StatFetcherCreator = \ - lambda x: FileStatsFetcher(os.getenv('JOSHUA_STAT_FILE', 'stats.json'), x) self.binary_chooser = OldBinaries() - - def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str): - def fdb_fetcher(tests: OrderedDict[str, TestDescription]): - from . import fdb - return fdb.FDBStatFetcher(cluster_file, app_dir, tests) - - self.stat_fetcher = fdb_fetcher + self.test_picker = TestPicker(self.test_path) def backup_sim_dir(self, seed: int): temp_dir = config.run_dir / str(self.uid) @@ -405,34 +419,37 @@ class TestRunner: count = 0 result: bool = True for file in test_files: + will_restart = count + 1 < len(test_files) binary = self.binary_chooser.choose_binary(file) - unseed_check = random.random() < config.unseed_check_ratio + unseed_check = config.random.random() < config.unseed_check_ratio if unseed_check and count != 0: # for restarting tests we will need to restore the sim2 after the first run self.backup_sim_dir(seed + count - 1) run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, - stats=test_picker.dump_stats()) + stats=test_picker.dump_stats(), will_restart=will_restart) result = result and run.success - test_picker.add_time(file, run.run_time) - if run.success and unseed_check and run.summary.unseed is not None: - self.restore_sim_dir(seed + count - 1) - run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, - stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed) - test_picker.add_time(file, run2.run_time) + test_picker.add_time(test_files[0], run.run_time) if not result: return False + if unseed_check and run.summary.unseed is not None: + if count != 0: + self.restore_sim_dir(seed + count - 1) + run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, + stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, + will_restart=will_restart) + test_picker.add_time(file, run2.run_time) + result = result and run.success count += 1 return result def run(self, stats: str | None) -> bool: - seed = random.randint(0, 2 ** 32 - 1) - test_picker = TestPicker(Path(self.test_path), self.stat_fetcher) + seed = config.random.randint(0, 2 ** 32 - 1) if stats is not None: - test_picker.load_stats(stats) + self.test_picker.load_stats(stats) else: - test_picker.fetch_stats() - test_files = test_picker.choose_test() - success = self.run_tests(test_files, seed, test_picker) + self.test_picker.fetch_stats() + test_files = self.test_picker.choose_test() + success = self.run_tests(test_files, seed, self.test_picker) if config.clean_up: shutil.rmtree(config.run_dir / str(self.uid)) return success diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 0ead14d090..dc1e6d8d74 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -13,7 +13,6 @@ import xml.sax.saxutils from pathlib import Path from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple -from xml.dom import minidom from test_harness.config import config @@ -46,7 +45,7 @@ class SummaryTree: # However, our xml is very simple and therefore serializing manually is easy enough attrs = [] for k, v in self.attributes.items(): - attrs.append('{}="{}"'.format(k, xml.sax.saxutils.escape(v))) + attrs.append('{}={}'.format(k, xml.sax.saxutils.quoteattr(v))) elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' ')) out.write(elem) if config.pretty_print: @@ -92,7 +91,7 @@ class ParseHandler: self.out = out self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict() - def add_handler(self, attr: Tuple[str, str], callback: ParserCallback) -> None: + def add_handler(self, attr: Tuple[str, Optional[str]], callback: ParserCallback) -> None: if attr in self.events: self.events[attr].append(callback) else: @@ -319,7 +318,7 @@ class TraceFiles: else: self.timestamps.append(ts) self.runs[ts] = [file] - self.timestamps.sort() + self.timestamps.sort(reverse=True) def __getitem__(self, idx: int) -> List[Path]: res = self.runs[self.timestamps[idx]] @@ -333,7 +332,8 @@ class TraceFiles: class Summary: def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None, was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None, - exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None): + exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None, + error_out: str = None, will_restart: bool = False): self.binary = binary self.runtime: float = runtime self.max_rss: int | None = max_rss @@ -353,17 +353,21 @@ class Summary: self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict() self.test_count: int = 0 self.tests_passed: int = 0 + self.error_out = error_out + self.stderr_severity: str = '40' + self.will_restart: bool = will_restart if uid is not None: self.out.attributes['TestUID'] = str(uid) if stats is not None: self.out.attributes['Statistics'] = stats self.out.attributes['JoshuaSeed'] = str(config.joshua_seed) + self.out.attributes['WillRestart'] = '1' if self.will_restart else '0' self.handler = ParseHandler(self.out) self.register_handlers() - def summarize(self, trace_dir: Path): + def summarize(self, trace_dir: Path, command: str): trace_files = TraceFiles(trace_dir) if len(trace_files) == 0: self.error = True @@ -371,14 +375,15 @@ class Summary: child = SummaryTree('NoTracesFound') child.attributes['Severity'] = '40' child.attributes['Path'] = str(trace_dir.absolute()) + child.attributes['Command'] = command self.out.append(child) + return for f in trace_files[0]: self.parse_file(f) self.done() def ok(self): - return not self.error and self.tests_passed == self.test_count and self.tests_passed >= 0\ - and self.test_end_found + return not self.error def done(self): if config.print_coverage: @@ -429,7 +434,38 @@ class Summary: child = SummaryTree('TestUnexpectedlyNotFinished') child.attributes['Severity'] = '40' self.out.append(child) + if self.error_out is not None and len(self.error_out) > 0: + if self.stderr_severity == '40': + self.error = True + lines = self.error_out.split('\n') + stderr_bytes = 0 + for line in lines: + remaining_bytes = config.max_stderr_bytes - stderr_bytes + if remaining_bytes > 0: + out_err = line[0:remaining_bytes] + ('...' if len(line) > remaining_bytes else '') + child = SummaryTree('StdErrOutput') + child.attributes['Severity'] = self.stderr_severity + child.attributes['Output'] = out_err + self.out.append(child) + stderr_bytes += len(line) + if stderr_bytes > config.max_stderr_bytes: + child = SummaryTree('StdErrOutputTruncated') + child.attributes['Severity'] = self.stderr_severity + child.attributes['BytesRemaining'] = stderr_bytes - config.max_stderr_bytes + self.out.append(child) + self.out.attributes['Ok'] = '1' if self.ok() else '0' + if not self.ok(): + reason = 'Unknown' + if self.error: + reason = 'ProducedErrors' + elif not self.test_end_found: + reason = 'TestDidNotFinish' + elif self.tests_passed == 0: + reason = 'NoTestsPassed' + elif self.test_count != self.tests_passed: + reason = 'Expected {} tests to pass, but only {} did'.format(self.test_count, self.tests_passed) + self.out.attributes['FailReason'] = reason def parse_file(self, file: Path): parser: Parser @@ -581,3 +617,8 @@ class Summary: self.out.append(child) self.handler.add_handler(('Type', 'BuggifySection'), buggify_section) self.handler.add_handler(('Type', 'FaultInjected'), buggify_section) + + def stderr_severity(attrs: Dict[str, str]): + if 'NewSeverity' in attrs: + self.stderr_severity = attrs['NewSeverity'] + self.handler.add_handler(('Type', 'StderrSeverity'), stderr_severity) diff --git a/contrib/TestHarness2/test_harness/version.py b/contrib/TestHarness2/test_harness/version.py index b5cc116f37..fe04206a8a 100644 --- a/contrib/TestHarness2/test_harness/version.py +++ b/contrib/TestHarness2/test_harness/version.py @@ -1,3 +1,9 @@ +from functools import total_ordering +from pathlib import Path +from typing import Tuple + + +@total_ordering class Version: def __init__(self): self.major: int = 0 @@ -7,24 +13,39 @@ class Version: def version_tuple(self): return self.major, self.minor, self.patch - def __eq__(self, other): - return self.version_tuple() == other.version_tuple() + def _compare(self, other) -> int: + lhs: Tuple[int, int, int] = self.version_tuple() + rhs: Tuple[int, int, int] + if isinstance(other, Version): + rhs = other.version_tuple() + else: + rhs = Version.parse(str(other)).version_tuple() + if lhs < rhs: + return -1 + elif lhs > rhs: + return 1 + else: + return 0 - def __lt__(self, other): - return self.version_tuple() < other.version_tuple() + def __eq__(self, other) -> bool: + return self._compare(other) == 0 - def __le__(self, other): - return self.version_tuple() <= other.version_tuple() - - def __gt__(self, other): - return self.version_tuple() > other.version_tuple() - - def __ge__(self, other): - return self.version_tuple() >= other.version_tuple() + def __lt__(self, other) -> bool: + return self._compare(other) < 0 def __hash__(self): return hash(self.version_tuple()) + def __str__(self): + return format('{}.{}.{}'.format(self.major, self.minor, self.patch)) + + @staticmethod + def of_binary(binary: Path): + parts = binary.name.split('-') + if len(parts) != 2: + return Version.max_version() + return Version.parse(parts[1]) + @staticmethod def parse(version: str): version_tuple = version.split('.') @@ -34,6 +55,7 @@ class Version: self.minor = int(version_tuple[1]) if len(version_tuple) > 2: self.patch = int(version_tuple[2]) + return self @staticmethod def max_version(): @@ -41,3 +63,4 @@ class Version: self.major = 2**32 - 1 self.minor = 2**32 - 1 self.patch = 2**32 - 1 + return self From 90863d07ad672bf7fbbafecb6d62e9bb60637b67 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sat, 20 Aug 2022 15:56:12 -0600 Subject: [PATCH 06/29] test harness now uses runtime to choose next test --- contrib/TestHarness2/stubs/fdb/__init__.pyi | 324 ------------------ contrib/TestHarness2/stubs/fdb/fdboptions.pyi | 113 ------ contrib/TestHarness2/test_harness/app.py | 9 +- contrib/TestHarness2/test_harness/config.py | 89 +++-- contrib/TestHarness2/test_harness/fdb.py | 84 +++-- contrib/TestHarness2/test_harness/results.py | 39 +++ contrib/TestHarness2/test_harness/run.py | 95 +++-- 7 files changed, 205 insertions(+), 548 deletions(-) delete mode 100644 contrib/TestHarness2/stubs/fdb/__init__.pyi delete mode 100644 contrib/TestHarness2/stubs/fdb/fdboptions.pyi create mode 100644 contrib/TestHarness2/test_harness/results.py diff --git a/contrib/TestHarness2/stubs/fdb/__init__.pyi b/contrib/TestHarness2/stubs/fdb/__init__.pyi deleted file mode 100644 index 4778c7ce92..0000000000 --- a/contrib/TestHarness2/stubs/fdb/__init__.pyi +++ /dev/null @@ -1,324 +0,0 @@ -import fdboptions - -from typing import Generic, TypeVar, Callable, List -from fdboptions import StreamingMode - -def api_version(version: int) -> None: ... - -def open(cluster_file: str = None, event_model: str = None) -> Database: - ... - -T = TypeVar('T') -options = fdboptions.NetworkOptions -StreamingMode = StreamingMode - - -class Future(Generic[T]): - def wait(self) -> T: - ... - - def is_ready(self) -> bool: - ... - - def block_until_ready(self) -> None: - ... - - def on_ready(self, callback: Callable[[Future[T]], None]) -> None: - ... - - def cancel(self): - ... - - @staticmethod - def wait_for_any(*futures: Future): - ... - - -class FutureString(Future[bytes]): - def as_foundationdb_key(self) -> bytes: - ... - - def as_foundationdb_value(self) -> bytes: - ... - - -class ValueType(FutureString): - def present(self) -> bool: - ... - -class KeyType(FutureString): - pass - - -Key = KeyType | bytes -Value = ValueType | bytes -Version = int - -class KVIter: - def __iter__(self) -> KVIter: - ... - - def __next__(self) -> KeyValue: - ... - - -class KeyValue: - key: Key - value: Value - - def __iter__(self) -> KVIter: - ... - - -class KeySelector: - @classmethod - def last_less_than(cls, key: Key) -> KeySelector: - ... - - @classmethod - def last_less_or_equal(cls, key: Key) -> KeySelector: - ... - - @classmethod - def first_greater_than(cls, key: Key) -> KeySelector: - ... - - @classmethod - def first_greater_or_equal(cls, key: Key) -> KeySelector: - ... - - def __add__(self, offset: int) -> KeySelector: - ... - - def __sub__(self, offset: int) -> KeySelector: - ... - - -class Error: - code: int - description: str - - -class Tenant: - def create_transaction(self) -> Transaction: - ... - - -class _SnapshotTransaction: - db: Database - def get(self, key: Key) -> bytes | None: - ... - - def get_key(self, key_selector: KeySelector) -> bytes: - ... - - def __getitem__(self, key: Key | slice) -> bytes: - ... - - def get_range(self, begin: Key, end: Key, limit: int = 0, reverse: bool = False, - streaming_mode: StreamingMode = StreamingMode.exact) -> KeyValue: - ... - - def get_range_startswith(self, prefix: bytes, - limit: int = 0, - reverse: bool = False, - streaming_mode: StreamingMode = StreamingMode.exact): - ... - - -class Transaction(_SnapshotTransaction): - options: fdboptions.TransactionOptions - snapshot: _SnapshotTransaction - - def set(self, key: Key, value: Value) -> None: - ... - - def clear(self, key: Key) -> None: - ... - - def clear_range(self, begin: Key, end: Key) -> None: - ... - - def clear_range_startswith(self, prefix: bytes) -> None: - ... - - def __setitem__(self, key: Key, value: Value) -> None: - ... - - def __delitem__(self, key: Key) -> None: - ... - - def add(self, key: Key, param: bytes) -> None: - ... - - def bit_and(self, key: Key, param: bytes) -> None: - ... - - def bit_or(self, key: Key, param: bytes) -> None: - ... - - def bit_xor(self, key: Key, param: bytes) -> None: - ... - - def max(self, key: Key, param: bytes) -> None: - ... - - def byte_max(self, key: Key, param: bytes) -> None: - ... - - def min(self, key: Key, param: bytes) -> None: - ... - - def byte_min(self, key: Key, param: bytes) -> None: - ... - - def set_versionstamped_key(self, key: Key, param: bytes) -> None: - ... - - def set_versionstamped_value(self, key: Key, param: bytes) -> None: - ... - - def commit(self) -> Future[None]: - ... - - def on_error(self, err: Error) -> Future[None]: - ... - - def reset(self) -> None: - ... - - def cancel(self) -> None: - ... - - def watch(self, key: Key) -> Future[None]: - ... - - def add_read_conflict_range(self, begin: Key, end: Key) -> None: - ... - - def add_read_conflict_key(self, key: Key) -> None: - ... - - def add_write_conflict_range(self, begin: Key, end: Key) -> None: - ... - - def add_write_conflict_key(self, key: Key) -> None: - ... - - def set_read_version(self, version: Version) -> None: - ... - - def get_read_version(self) -> Version: - ... - - def get_committed_version(self) -> Version: - ... - - def get_versionstamp(self) -> FutureString: - ... - - -class Database: - options: fdboptions.DatabaseOptions - - def create_transaction(self) -> Transaction: - ... - - def open_tenant(self, tenant_name: str) -> Tenant: - ... - - def get(self, key: Key) -> bytes | None: - ... - - def get_key(self, key_selector: KeySelector) -> bytes: - ... - - def clear(self, ): - ... - - def __getitem__(self, key: Key | slice) -> bytes: - ... - - def __setitem__(self, key: Key, value: Value) -> None: - ... - - def __delitem__(self, key: Key) -> None: - ... - - def get_range(self, begin: Key, end: Key, limit: int = 0, reverse: bool = False, - streaming_mode: StreamingMode = StreamingMode.exact) -> KeyValue: - ... - - -class Subspace: - def __init__(self, **kwargs): - ... - - def key(self) -> bytes: - ... - - def pack(self, tuple: tuple = tuple()) -> bytes: - ... - - def pack_with_versionstamp(self, tuple: tuple) -> bytes: - ... - - def unpack(self, key: bytes) -> tuple: - ... - - def range(self, tuple: tuple = tuple()) -> slice: - ... - - def contains(self, key: bytes) -> bool: - ... - - def subspace(self, tuple: tuple) -> Subspace: - ... - - def __getitem__(self, item) -> Subspace: - ... - - -class DirectoryLayer: - def __init__(self, *kwargs): - ... - - def create_or_open(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: - ... - - def open(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: - ... - - def create(self, tr: Transaction, path: tuple | str, layer: bytes = None) -> DirectorySubspace: - ... - - def move(self, tr: Transaction, old_path: tuple | str, new_path: tuple | str) -> DirectorySubspace: - ... - - def remove(self, tr: Transaction, path: tuple | str) -> None: - ... - - def remove_if_exists(self, tr: Transaction, path: tuple | str) -> None: - ... - - def list(self, tr: Transaction, path: tuple | str = ()) -> List[str]: - ... - - def exists(self, tr: Transaction, path: tuple | str) -> bool: - ... - - def get_layer(self) -> bytes: - ... - - def get_path(self) -> tuple: - ... - - -class DirectorySubspace(DirectoryLayer, Subspace): - def move_to(self, tr: Transaction, new_path: tuple | str) -> DirectorySubspace: - ... - -directory: DirectoryLayer - -def transactional(*tr_args, **tr_kwargs) -> Callable: - ... \ No newline at end of file diff --git a/contrib/TestHarness2/stubs/fdb/fdboptions.pyi b/contrib/TestHarness2/stubs/fdb/fdboptions.pyi deleted file mode 100644 index fadfd0ec0c..0000000000 --- a/contrib/TestHarness2/stubs/fdb/fdboptions.pyi +++ /dev/null @@ -1,113 +0,0 @@ -import enum - - -class StreamingMode(enum.Enum): - want_all = -2 - iterator = -1 - exact = 0 - small = 1 - medium = 2 - large = 3 - serial = 4 - -class NetworkOptions(enum.Enum): - local_address = 10 - cluster_file = 20 - trace_enable = 30 - trace_roll_size = 31 - trace_max_logs_size = 32 - trace_log_group = 33 - trace_format = 34 - trace_clock_source = 35 - trace_file_identifier = 36 - trace_partial_file_suffix = 39 - knob = 40 - TLS_plugin = 41 - TLS_cert_bytes = 42 - TLS_cert_path = 43 - TLS_key_bytes = 45 - TLS_key_path = 46 - TLS_verify_peers = 47 - Buggify_enable = 48 - Buggify_disable = 49 - Buggify_section_activated_probability = 50 - Buggify_section_fired_probability = 51 - TLS_ca_bytes = 52 - TLS_ca_path = 53 - TLS_password = 54 - disable_multi_version_client_api = 60 - callbacks_on_external_threads = 61 - external_client_library = 62 - external_client_directory = 63 - disable_local_client = 64 - client_threads_per_version = 65 - disable_client_statistics_logging = 70 - enable_slow_task_profiling = 71 - enable_run_loop_profiling = 71 - client_buggify_enable = 80 - client_buggify_disable = 81 - client_buggify_section_activated_probability = 82 - client_buggify_section_fired_probability = 83 - distributed_client_tracer = 90 - -class DatabaseOptions: - location_cache_size = 10 - max_watches = 20 - machine_id = 21 - datacenter_id = 22 - snapshot_ryw_enable = 26 - snapshot_ryw_disable = 27 - transaction_logging_max_field_length = 405 - transaction_timeout = 500 - transaction_retry_limit = 501 - transaction_max_retry_delay = 502 - transaction_size_limit = 503 - transaction_causal_read_risky = 504 - transaction_include_port_in_address = 505 - transaction_bypass_unreadable = 700 - use_config_database = 800 - test_causal_read_risky = 900 - - -class TransactionOptions: - causal_write_risky = 10 - causal_read_risky = 20 - causal_read_disable = 21 - include_port_in_address = 23 - next_write_no_write_conflict_range = 30 - read_your_writes_disable = 51 - read_ahead_disable = 52 - durability_datacenter = 110 - durability_risky = 120 - durability_dev_null_is_web_scale = 130 - priority_system_immediate = 200 - priority_batch = 201 - initialize_new_database = 300 - access_system_keys = 301 - read_system_keys = 302 - raw_access = 303 - debug_retry_logging = 401 - transaction_logging_enable = 402 - debug_transaction_identifier = 403 - log_transaction = 404 - transaction_logging_max_field_length = 405 - server_request_tracing = 406 - timeout = 500 - retry_limit = 501 - max_retry_delay = 502 - size_limit = 503 - snapshot_ryw_enable = 600 - snapshot_ryw_disable = 601 - lock_aware = 700 - used_during_commit_protection_disable = 701 - read_lock_aware = 702 - use_provisional_proxies = 711 - report_conflicting_keys = 712 - special_key_space_relaxed = 713 - special_key_space_enable_writes = 714 - tag = 800 - auto_throttle_tag = 801 - span_parent = 900 - expensive_clear_cost_estimation_enable = 1000 - bypass_unreadable = 1100 - use_grv_cache = 1101 diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py index 27edc0f065..3e300c6bf4 100644 --- a/contrib/TestHarness2/test_harness/app.py +++ b/contrib/TestHarness2/test_harness/app.py @@ -6,21 +6,14 @@ from test_harness.config import config from test_harness.run import TestRunner from test_harness.summarize import SummaryTree - if __name__ == '__main__': try: parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.build_arguments(parser) - # initialize arguments - parser.add_argument('--joshua-dir', type=str, help='Where to write FDB data to', required=False) - parser.add_argument('-C', '--cluster-file', type=str, help='Path to fdb cluster file', required=False) - parser.add_argument('--stats', type=str, - help='A base64 encoded list of statistics (used to reproduce runs)', - required=False) args = parser.parse_args() config.extract_args(args) test_runner = TestRunner() - if not test_runner.run(args.stats): + if not test_runner.run(): exit(1) except Exception as e: _, _, exc_traceback = sys.exc_info() diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index ddaf24a077..1d527a9899 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -7,7 +7,7 @@ import os import random from enum import Enum from pathlib import Path -from typing import List, Any, OrderedDict +from typing import List, Any, OrderedDict, Dict class BuggifyOptionValue(Enum): @@ -51,6 +51,8 @@ class ConfigValue: if 'short_name' in kwargs: short_name = kwargs['short_name'] del kwargs['short_name'] + if 'action' in kwargs and kwargs['action'] in ['store_true', 'store_false']: + del kwargs['type'] long_name = long_name.replace('_', '-') if short_name is None: # line below is useful for debugging @@ -71,6 +73,15 @@ class ConfigValue: class Config: def __init__(self): self.random = random.Random() + self.cluster_file: str | None = None + self.cluster_file_args = {'short_name': 'C', 'type': str, 'help': 'Path to fdb cluster file', 'required': False, + 'env_name': 'JOSHUA_CLUSTER_FILE'} + self.joshua_dir: str | None = None + self.joshua_dir_args = {'type': str, 'help': 'Where to write FDB data to', 'required': False, + 'env_name': 'JOSHUA_APP_DIR'} + self.stats: str | None = None + self.stats_args = {'type': str, 'help': 'A base64 encoded list of statistics (used to reproduce runs)', + 'required': False} self.kill_seconds: int = 30 * 60 self.kill_seconds_args = {'help': 'Timeout for individual test'} self.buggify_on_ratio: float = 0.8 @@ -91,7 +102,7 @@ class Config: self.max_warnings_args = {'short_name': 'W'} self.max_errors: int = 10 self.max_errors_args = {'short_name': 'E'} - self.old_binaries_path: Path = Path('/opt/joshua/global_data/oldBinaries') + self.old_binaries_path: Path = Path('/app/deploy/global_data/oldBinaries/') self.old_binaries_path_args = {'help': 'Path to the directory containing the old fdb binaries'} self.use_valgrind: bool = False self.use_valgrind_args = {'action': 'store_true'} @@ -99,12 +110,11 @@ class Config: self.buggify_args = {'short_name': 'b', 'choices': ['on', 'off', 'random']} self.pretty_print: bool = False self.pretty_print_args = {'short_name': 'P', 'action': 'store_true'} - self.clean_up = True + self.clean_up: bool = True self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'} self.run_dir: Path = Path('tmp') - self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1)))) - self.random.seed(self.joshua_seed, version=2) - self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'} + self.joshua_seed: int = random.randint(0, 2 ** 32 - 1) + self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed', 'env_name': 'JOSHUA_SEED'} self.print_coverage = False self.print_coverage_args = {'action': 'store_true'} self.binary = Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver') @@ -122,46 +132,75 @@ class Config: self.max_stderr_bytes: int = 1000 self.write_stats: bool = True self.read_stats: bool = True - self.config_map = self._build_map() + self._env_names: Dict[str, str] = {} + self._config_map = self._build_map() + self._read_env() + self.random.seed(self.joshua_seed, version=2) - def _build_map(self): + def _get_env_name(self, var_name: str) -> str: + return self._env_names.get(var_name, 'TH_{}'.format(var_name.upper())) + + def dump(self): + for attr in dir(self): + obj = getattr(self, attr) + if attr == 'random' or attr.startswith('_') or callable(obj) or attr.endswith('_args'): + continue + print('config.{}: {} = {}'.format(attr, type(obj), obj)) + + def _build_map(self) -> OrderedDict[str, ConfigValue]: config_map: OrderedDict[str, ConfigValue] = collections.OrderedDict() for attr in dir(self): obj = getattr(self, attr) - if attr.startswith('_') or callable(obj): + if attr == 'random' or attr.startswith('_') or callable(obj): continue if attr.endswith('_args'): name = attr[0:-len('_args')] assert name in config_map assert isinstance(obj, dict) for k, v in obj.items(): - if k == 'action' and v in ['store_true', 'store_false']: - # we can't combine type and certain actions - del config_map[name].kwargs['type'] - config_map[name].kwargs[k] = v + if k == 'env_name': + self._env_names[name] = v + else: + config_map[name].kwargs[k] = v else: + # attribute_args has to be declared after the attribute + assert attr not in config_map val_type = type(obj) - env_name = 'TH_{}'.format(attr.upper()) - new_val = os.getenv(env_name) - try: - if new_val is not None: - obj = val_type(new_val) - except: - pass kwargs = {'type': val_type, 'default': obj} config_map[attr] = ConfigValue(attr, **kwargs) return config_map + def _read_env(self): + for attr in dir(self): + obj = getattr(self, attr) + if attr == 'random' or attr.startswith('_') or attr.endswith('_args') or callable(obj): + continue + env_name = self._get_env_name(attr) + attr_type = self._config_map[attr].kwargs['type'] + assert type(None) != attr_type + e = os.getenv(env_name) + if e is not None: + self.__setattr__(attr, attr_type(e)) + def build_arguments(self, parser: argparse.ArgumentParser): - for val in self.config_map.values(): + for val in self._config_map.values(): val.add_to_args(parser) def extract_args(self, args: argparse.Namespace): - for val in self.config_map.values(): + for val in self._config_map.values(): k, v = val.get_value(args) - config.__setattr__(k, v) - if k == 'joshua_seed': - self.random.seed(self.joshua_seed, version=2) + if v is not None: + config.__setattr__(k, v) + self.random.seed(self.joshua_seed, version=2) config = Config() + +if __name__ == '__main__': + # test the config setup + parser = argparse.ArgumentParser('TestHarness Config Tester', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + config.build_arguments(parser) + args = parser.parse_args() + config.extract_args(args) + config.dump() diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index 35f65c3544..eee8eb5beb 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -1,40 +1,72 @@ from __future__ import annotations -from typing import OrderedDict +from typing import OrderedDict, Tuple +import collections import fdb import struct from test_harness.run import StatFetcher, TestDescription +from test_harness.config import config +from test_harness.summarize import SummaryTree + +fdb.api_version(630) + + +def str_to_tuple(s: str | None): + if s is None: + return s + res = s.split(',') + return tuple(res) + + +class TestStatistics: + def __init__(self, runtime: int, run_count: int): + self.runtime: int = runtime + self.run_count: int = run_count + + +class Statistics: + def __init__(self, cluster_file: str | None, joshua_dir: Tuple[str, ...]): + self.db: fdb.Database = fdb.open(cluster_file) + self.stats_dir: fdb.DirectorySubspace = self.open_stats_dir(self.db, joshua_dir) + self.stats: OrderedDict[str, TestStatistics] = self.read_stats_from_db(self.db) + + @fdb.transactional + def open_stats_dir(self, tr, app_dir: Tuple[str]) -> fdb.DirectorySubspace: + stats_dir = app_dir + ('runtime_stats',) + return fdb.directory.create_or_open(tr, stats_dir) + + @fdb.transactional + def read_stats_from_db(self, tr) -> OrderedDict[str, TestStatistics]: + result = collections.OrderedDict() + for k, v in tr[self.stats_dir.range()]: + test_name = self.stats_dir.unpack(k)[0] + runtime, run_count = struct.unpack(' None: + key = self.stats_dir.pack((test_name,)) + tr.add(key, struct.pack(' None: + self._write_runtime(self.db, test_name, time) class FDBStatFetcher(StatFetcher): - def __init__(self, cluster_file: str | None, app_dir: str, tests: OrderedDict[str, TestDescription]): + def __init__(self, tests: OrderedDict[str, TestDescription], + joshua_dir: Tuple[str] = str_to_tuple(config.joshua_dir)): super().__init__(tests) - fdb.api_version(630) - self.db: fdb.Database = fdb.open(cluster_file) - self.stats_dir: fdb.DirectorySubspace = self.open_stats_dir(self.db, app_dir) - - @fdb.transactional - def open_stats_dir(self, tr, app_dir: str) -> fdb.DirectorySubspace: - app_dir_path = app_dir.split(',') - app_dir_path.append('runtime_stats') - return fdb.directory.create_or_open(tr, tuple(app_dir_path)) - - @fdb.transactional - def read_stats_from_db(self, tr): - for k, v in tr[self.stats_dir.range()]: - test_name = self.stats_dir.unpack(k)[0] - if test_name in self.tests.keys(): - self.tests[test_name] = struct.unpack(' None: + def add_time(self, test_file: Path, run_time: int, out: SummaryTree) -> None: # getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother - test_name = None + test_name: str | None = None + test_desc: TestDescription | None = None for name, test in self.tests.items(): for p in test.paths: - if p.absolute() == test_file.absolute(): - test_name = name + test_files: List[Path] + if self.restart_test.match(p.name): + test_files = self.list_restart_files(p) + else: + test_files = [p] + for file in test_files: + if file.absolute() == test_file.absolute(): + test_name = name + test_desc = test + break + if test_name is not None: break if test_name is not None: break - # assert test_name is not None - # self.fetcher.add_run_time(test_name, run_time) + assert test_name is not None and test_desc is not None + self.stat_fetcher.add_run_time(test_name, run_time, out) + out.attributes['TotalTestTime'] = str(test_desc.total_runtime) + out.attributes['TestRunCount'] = str(test_desc.num_runs) def dump_stats(self) -> str: res = array.array('I') @@ -120,8 +116,7 @@ class TestPicker: return base64.standard_b64encode(res.tobytes()).decode('utf-8') def fetch_stats(self): - # self.fetcher.read_stats() - pass + self.stat_fetcher.read_stats() def load_stats(self, serialized: str): times = array.array('I') @@ -198,9 +193,7 @@ class TestPicker: candidates: List[TestDescription] = [] for _, v in self.tests.items(): this_time = v.total_runtime * v.priority - if min_runtime is None: - min_runtime = this_time - if this_time < min_runtime: + if min_runtime is None or this_time < min_runtime: min_runtime = this_time candidates = [v] elif this_time == min_runtime: @@ -315,7 +308,7 @@ class TestRun: self.old_binary_path: Path = config.old_binaries_path self.buggify_enabled: bool = config.random.random() < config.buggify_on_ratio self.fault_injection_enabled: bool = True - self.trace_format = config.trace_format + self.trace_format: str | None = config.trace_format if Version.of_binary(self.binary) < "6.1.0": self.trace_format = None self.temp_path = config.run_dir / str(self.uid) @@ -379,14 +372,14 @@ class TestRun: did_kill = True resources.stop() resources.join() - self.run_time = round(resources.time()) + # we're rounding times up, otherwise we will prefer running very short tests (<1s) + self.run_time = math.ceil(resources.time()) self.summary.runtime = resources.time() self.summary.max_rss = resources.max_rss self.summary.was_killed = did_kill self.summary.valgrind_out_file = valgrind_file self.summary.error_out = err_out self.summary.summarize(self.temp_path, ' '.join(command)) - self.summary.out.dump(sys.stdout) return self.summary.ok() @@ -428,7 +421,8 @@ class TestRunner: run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), will_restart=will_restart) result = result and run.success - test_picker.add_time(test_files[0], run.run_time) + test_picker.add_time(test_files[0], run.run_time, run.summary.out) + run.summary.out.dump(sys.stdout) if not result: return False if unseed_check and run.summary.unseed is not None: @@ -437,17 +431,14 @@ class TestRunner: run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, will_restart=will_restart) - test_picker.add_time(file, run2.run_time) + test_picker.add_time(file, run2.run_time, run.summary.out) + run.summary.out.dump(sys.stdout) result = result and run.success count += 1 return result - def run(self, stats: str | None) -> bool: + def run(self) -> bool: seed = config.random.randint(0, 2 ** 32 - 1) - if stats is not None: - self.test_picker.load_stats(stats) - else: - self.test_picker.fetch_stats() test_files = self.test_picker.choose_test() success = self.run_tests(test_files, seed, self.test_picker) if config.clean_up: From 48083073541558ed70dbae389c70f9dc88033dad Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sat, 20 Aug 2022 20:01:09 -0600 Subject: [PATCH 07/29] collect and report on code probe information --- contrib/TestHarness2/test_harness/config.py | 11 ++- contrib/TestHarness2/test_harness/fdb.py | 66 +++++++++++-- contrib/TestHarness2/test_harness/results.py | 92 +++++++++++++++---- .../TestHarness2/test_harness/summarize.py | 13 ++- 4 files changed, 158 insertions(+), 24 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 1d527a9899..acc1e13622 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -119,8 +119,11 @@ class Config: self.print_coverage_args = {'action': 'store_true'} self.binary = Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver') self.binary_args = {'help': 'Path to executable'} + self.hit_per_runs_ratio: int = 20000 + self.hit_per_runs_ratio_args = {'help': 'How many test runs should hit each code probe at least once'} self.output_format: str = 'xml' - self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']} + self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml'], + 'help': 'What format TestHarness should produce'} self.include_test_files: str = r'.*' self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'} self.exclude_test_files: str = r'.^' @@ -129,6 +132,12 @@ class Config: self.include_test_names_args = {'help': 'Only consider tests whose names match against the given regex'} self.exclude_test_names: str = r'.^' self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'} + self.details: bool = False + self.details_args = {'help': 'Print detailed results', 'short_name': 'c'} + self.cov_include_files: str = r'.*' + self.cov_include_files_args = {'help': 'Only consider coverage traces that originated in files matching regex'} + self.cov_exclude_files: str = r'.^' + self.cov_exclude_files_args = {'help': 'Ignore coverage traces that originated in files matching regex'} self.max_stderr_bytes: int = 1000 self.write_stats: bool = True self.read_stats: bool = True diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index eee8eb5beb..0eed36bae3 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import OrderedDict, Tuple +from typing import OrderedDict, Tuple, List import collections import fdb @@ -8,7 +8,7 @@ import struct from test_harness.run import StatFetcher, TestDescription from test_harness.config import config -from test_harness.summarize import SummaryTree +from test_harness.summarize import SummaryTree, Coverage fdb.api_version(630) @@ -20,6 +20,59 @@ def str_to_tuple(s: str | None): return tuple(res) +fdb_db = None + + +def open_db(cluster_file: str | None): + global fdb_db + if fdb_db is None: + fdb_db = fdb.open(cluster_file) + return fdb_db + +def chunkify(iterable, sz: int): + count = 0 + res = [] + for item in iterable: + res.append(item) + count += 1 + if count >= sz: + yield res + res = [] + count = 0 + if len(res) > 0: + yield res + + +@fdb.transactional +def write_coverage_chunk(tr, path: Tuple[str, ...], coverage: List[Tuple[Coverage, bool]]): + cov_dir = fdb.directory.create_or_open(tr, path) + for cov, covered in coverage: + tr.add(cov_dir.pack((cov.file, cov.line, cov.comment)), struct.pack(' OrderedDict[Coverage, int]: + res = collections.OrderedDict() + cov_dir = fdb.directory.create_or_open(tr, cov_path) + for k, v in tr[cov_dir.range()]: + file, line, comment = cov_dir.unpack(k) + count = struct.unpack(' OrderedDict[Coverage, int]: + db = open_db(cluster_file) + return _read_coverage(db, cov_path) + + class TestStatistics: def __init__(self, runtime: int, run_count: int): self.runtime: int = runtime @@ -28,12 +81,12 @@ class TestStatistics: class Statistics: def __init__(self, cluster_file: str | None, joshua_dir: Tuple[str, ...]): - self.db: fdb.Database = fdb.open(cluster_file) - self.stats_dir: fdb.DirectorySubspace = self.open_stats_dir(self.db, joshua_dir) + self.db = open_db(cluster_file) + self.stats_dir = self.open_stats_dir(self.db, joshua_dir) self.stats: OrderedDict[str, TestStatistics] = self.read_stats_from_db(self.db) @fdb.transactional - def open_stats_dir(self, tr, app_dir: Tuple[str]) -> fdb.DirectorySubspace: + def open_stats_dir(self, tr, app_dir: Tuple[str]): stats_dir = app_dir + ('runtime_stats',) return fdb.directory.create_or_open(tr, stats_dir) @@ -47,11 +100,12 @@ class Statistics: return result @fdb.transactional - def _write_runtime(self, tr: fdb.Transaction, test_name: str, time: int) -> None: + def _write_runtime(self, tr, test_name: str, time: int) -> None: key = self.stats_dir.pack((test_name,)) tr.add(key, struct.pack(' None: + assert self.db is not None self._write_runtime(self.db, test_name, time) diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index d37dbfa35a..064d1af4d2 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -1,39 +1,99 @@ from __future__ import annotations +import re import sys -from typing import List, Tuple +from typing import List, Tuple, OrderedDict -from test_harness.summarize import SummaryTree +from test_harness.summarize import SummaryTree, Coverage from test_harness.config import config import argparse import test_harness.fdb +class GlobalStatistics: + def __init__(self): + self.total_probes_hit: int = 0 + self.total_cpu_time: int = 0 + self.total_test_runs: int = 0 + self.total_missed_probes: int = 0 + + class EnsembleResults: def __init__(self, cluster_file: str | None, ensemble_id: str): + self.global_statistics = GlobalStatistics() self.fdb_path = ('joshua', 'ensembles', 'results', 'application', ensemble_id) + self.coverage_path = self.fdb_path + ('coverage',) self.statistics = test_harness.fdb.Statistics(cluster_file, self.fdb_path) - self.out = SummaryTree('EnsembleResults') - stats: List[Tuple[str, int, int]] = [] + coverage_dict: OrderedDict[Coverage, int] = test_harness.fdb.read_coverage(cluster_file, self.coverage_path) + self.coverage: List[Tuple[Coverage, int]] = [] + self.min_coverage_hit: int | None = None + self.ratio = self.global_statistics.total_test_runs / config.hit_per_runs_ratio + for cov, count in coverage_dict.items(): + if re.search(config.cov_include_files, cov.file) is None: + continue + if re.search(config.cov_exclude_files, cov.file) is not None: + continue + self.global_statistics.total_probes_hit += count + self.coverage.append((cov, count)) + if count <= self.ratio: + self.global_statistics.total_missed_probes += 1 + if self.min_coverage_hit is None or self.min_coverage_hit > count: + self.min_coverage_hit = count + self.coverage.sort(key=lambda x: (x[1], x[0].file, x[0].line)) + self.stats: List[Tuple[str, int, int]] = [] for k, v in self.statistics.stats.items(): - stats.append((k, v.runtime, v.run_count)) - stats.sort(key=lambda x: x[1], reverse=True) - for k, runtime, run_count in stats: - child = SummaryTree('Test') - child.attributes['Name'] = k - child.attributes['Runtime'] = str(runtime) - child.attributes['RunCount'] = str(run_count) - self.out.append(child) + self.global_statistics.total_test_runs += v.run_count + self.global_statistics.total_cpu_time += v.runtime + self.stats.append((k, v.runtime, v.run_count)) + self.stats.sort(key=lambda x: x[1], reverse=True) + self.coverage_ok: bool = self.min_coverage_hit is not None + if self.coverage_ok: + self.coverage_ok = self.min_coverage_hit > self.ratio + + def dump(self): + errors = 0 + out = SummaryTree('EnsembleResults') + out.attributes['TotalRunTime'] = str(self.global_statistics.total_cpu_time) + out.attributes['TotalTestRuns'] = str(self.global_statistics.total_test_runs) + out.attributes['TotalProbesHit'] = str(self.global_statistics.total_probes_hit) + out.attributes['MinProbeHit'] = str(self.min_coverage_hit) + out.attributes['TotalProbes'] = str(len(self.coverage)) + out.attributes['MissedProbes'] = str(self.global_statistics.total_missed_probes) + + for cov, count in self.coverage: + severity = 10 if count > self.ratio else 40 + if severity == 40: + errors += 1 + if (severity == 40 and errors <= config.max_errors) or config.details: + child = SummaryTree('CodeProbe') + child.attributes['Severity'] = str(severity) + child.attributes['File'] = cov.file + child.attributes['Line'] = str(cov.line) + child.attributes['Comment'] = cov.comment + child.attributes['HitCount'] = str(count) + out.append(child) + + if config.details: + for k, runtime, run_count in self.stats: + child = SummaryTree('Test') + child.attributes['Name'] = k + child.attributes['Runtime'] = str(runtime) + child.attributes['RunCount'] = str(run_count) + out.append(child) + if errors > 0: + out.attributes['Errors'] = str(errors) + out.dump(sys.stdout) if __name__ == '__main__': parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-C', '--cluster-file', required=False, help='Path to cluster file') - parser.add_argument('-o', '--output-format', default='json', choices=['json', 'xml'], help='Format of the output') + config.build_arguments(parser) parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for') args = parser.parse_args() + config.extract_args(args) config.pretty_print = True config.output_format = args.output_format - results = EnsembleResults(args.cluster_file, args.ensemble_id) - results.out.dump(sys.stdout) + results = EnsembleResults(config.cluster_file, args.ensemble_id) + results.dump() + exit(0 if results.coverage_ok else 1) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index dc1e6d8d74..ad70a7a2ed 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -3,6 +3,7 @@ from __future__ import annotations import collections import inspect import json +import os import re import sys import traceback @@ -44,6 +45,11 @@ class SummaryTree: # minidom doesn't support omitting the xml declaration which is a problem for joshua # However, our xml is very simple and therefore serializing manually is easy enough attrs = [] + print_width = 120 + try: + print_width, _ = os.get_terminal_size() + except OSError: + pass for k, v in self.attributes.items(): attrs.append('{}={}'.format(k, xml.sax.saxutils.quoteattr(v))) elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' ')) @@ -52,7 +58,7 @@ class SummaryTree: curr_line_len = len(elem) for i in range(len(attrs)): attr_len = len(attrs[i]) - if i == 0 or attr_len + curr_line_len + 1 <= 120: + if i == 0 or attr_len + curr_line_len + 1 <= print_width: if i != 0: out.write(' ') out.write(attrs[i]) @@ -381,6 +387,11 @@ class Summary: for f in trace_files[0]: self.parse_file(f) self.done() + if config.joshua_dir is not None: + import test_harness.fdb + test_harness.fdb.write_coverage(config.cluster_file, + test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage',), + self.coverage) def ok(self): return not self.error From 7c53e8ee81568e987259030723d3cf4c5cd90e7d Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sun, 21 Aug 2022 08:23:51 -0600 Subject: [PATCH 08/29] Added RunningUnitTest to output --- contrib/TestHarness2/test_harness/summarize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index ad70a7a2ed..4db6ae154a 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -629,6 +629,13 @@ class Summary: self.handler.add_handler(('Type', 'BuggifySection'), buggify_section) self.handler.add_handler(('Type', 'FaultInjected'), buggify_section) + def running_unit_test(attrs: Dict[str, str]): + child = SummaryTree('RunningUnitTest') + child.attributes['Name'] = attrs['Name'] + child.attributes['File'] = attrs['File'] + child.attributes['Line'] = attrs['Line'] + self.handler.add_handler(('Type', 'RunningUnitTest'), running_unit_test) + def stderr_severity(attrs: Dict[str, str]): if 'NewSeverity' in attrs: self.stderr_severity = attrs['NewSeverity'] From cd7af3f7c83cd660ecd6513c4ba9cf919b7608f0 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sun, 21 Aug 2022 10:42:24 -0600 Subject: [PATCH 09/29] Also print test errors in results We now attempt to import joshua.joshua_model. If this succeeds test_harness.results will also print test errors --- contrib/TestHarness2/test_harness/config.py | 5 ++ contrib/TestHarness2/test_harness/joshua.py | 63 +++++++++++++++++++ contrib/TestHarness2/test_harness/results.py | 41 ++++++++++-- contrib/TestHarness2/test_harness/run.py | 19 +++++- .../TestHarness2/test_harness/summarize.py | 25 +++++--- 5 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 contrib/TestHarness2/test_harness/joshua.py diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index acc1e13622..b4b4b34113 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -146,6 +146,11 @@ class Config: self._read_env() self.random.seed(self.joshua_seed, version=2) + def change_default(self, attr: str, default_val): + assert attr in self._config_map, 'Unknown config attribute {}'.format(attr) + self.__setattr__(attr, default_val) + self._config_map[attr].kwargs['default'] = default_val + def _get_env_name(self, var_name: str) -> str: return self._env_names.get(var_name, 'TH_{}'.format(var_name.upper())) diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py new file mode 100644 index 0000000000..6411bb6cc3 --- /dev/null +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import sys +import xml.sax +import xml.sax.handler +from typing import List + +from joshua import joshua_model +from test_harness.config import config +from test_harness.summarize import SummaryTree + + +class ToSummaryTree(xml.sax.handler.ContentHandler): + def __init__(self): + super().__init__() + self.root: SummaryTree | None = None + self.stack: List[SummaryTree] = [] + + def result(self) -> SummaryTree: + assert len(self.stack) == 0 and self.root is not None, 'Parse Error' + return self.root + + def startElement(self, name, attrs): + new_child = SummaryTree(name) + for k, v in attrs.items(): + new_child.attributes[k] = v + self.stack.append(new_child) + + def endElement(self, name): + closed = self.stack.pop() + assert closed.name == name + if len(self.stack) == 0: + self.root = closed + else: + self.stack[-1].children.append(closed) + + +def print_errors(ensemble_id: str): + joshua_model.open(config.cluster_file) + properties = joshua_model.get_ensemble_properties(ensemble_id) + compressed = properties["compressed"] if "compressed" in properties else False + for rec in joshua_model.tail_results(ensemble_id, errors_only=(not config.details), compressed=compressed): + if len(rec) == 5: + version_stamp, result_code, host, seed, output = rec + elif len(rec) == 4: + version_stamp, result_code, host, output = rec + seed = None + elif len(rec) == 3: + version_stamp, result_code, output = rec + host = None + seed = None + elif len(rec) == 2: + version_stamp, seed = rec + output = str(joshua_model.fdb.tuple.unpack(seed)[0]) + "\n" + result_code = None + host = None + seed = None + else: + raise Exception("Unknown result format") + summary = ToSummaryTree() + xml.sax.parseString(output, summary) + res = summary.result() + res.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index 064d1af4d2..d034aa70bc 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -51,7 +51,7 @@ class EnsembleResults: if self.coverage_ok: self.coverage_ok = self.min_coverage_hit > self.ratio - def dump(self): + def dump(self, prefix: str): errors = 0 out = SummaryTree('EnsembleResults') out.attributes['TotalRunTime'] = str(self.global_statistics.total_cpu_time) @@ -83,17 +83,50 @@ class EnsembleResults: out.append(child) if errors > 0: out.attributes['Errors'] = str(errors) - out.dump(sys.stdout) + out.dump(sys.stdout, prefix=prefix, new_line=config.pretty_print) + + +def write_header(ensemble_id: str): + if config.output_format == 'json': + if config.pretty_print: + print('{') + print(' {}: {}'.format('ID', ensemble_id)) + else: + sys.stdout.write('{{{}: {}'.format('ID', ensemble_id)) + elif config.output_format == 'xml': + sys.stdout.write(''.format(ensemble_id)) + if config.pretty_print: + sys.stdout.write('\n') + else: + assert False, 'unknown output format {}'.format(config.output_format) + + +def write_footer(): + if config.output_format == 'json': + sys.stdout.write('}') + elif config.output_format == 'xml': + sys.stdout.write('') + else: + assert False, 'unknown output format {}'.format(config.output_format) if __name__ == '__main__': parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + config.change_default('pretty_print', True) config.build_arguments(parser) parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for') args = parser.parse_args() config.extract_args(args) - config.pretty_print = True config.output_format = args.output_format + write_header(args.ensemble_id) + try: + import test_harness.joshua + test_harness.joshua.print_errors(args.ensemble_id) + except ModuleNotFoundError: + child = SummaryTree('JoshuaNotFound') + child.attributes['Severity'] = '30' + child.attributes['Message'] = 'Could not import Joshua -- set PYTHONPATH to joshua checkout dir' + child.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) results = EnsembleResults(config.cluster_file, args.ensemble_id) - results.dump() + results.dump(' ' if config.pretty_print else '') exit(0 if results.coverage_ok else 1) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index e6493c1bb1..ce05dbf8b1 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -383,6 +383,19 @@ class TestRun: return self.summary.ok() +def decorate_summary(out: SummaryTree, test_file: Path, seed: int, buggify: bool): + """Sometimes a test can crash before ProgramStart is written to the traces. These + tests are then hard to reproduce (they can be reproduced through TestHarness but + require the user to run in the joshua docker container). To account for this we + will write the necessary information into the attributes if it is missing.""" + if 'TestFile' not in out.attributes: + out.attributes['TestFile'] = str(test_file) + if 'RandomSeed' not in out.attributes: + out.attributes['RandomSeed'] = str(seed) + if 'BuggifyEnabled' not in out.attributes: + out.attributes['BuggifyEnabled'] = '1' if buggify else '0' + + class TestRunner: def __init__(self): self.uid = uuid.uuid4() @@ -422,6 +435,7 @@ class TestRunner: stats=test_picker.dump_stats(), will_restart=will_restart) result = result and run.success test_picker.add_time(test_files[0], run.run_time, run.summary.out) + decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled) run.summary.out.dump(sys.stdout) if not result: return False @@ -432,8 +446,9 @@ class TestRunner: stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, will_restart=will_restart) test_picker.add_time(file, run2.run_time, run.summary.out) - run.summary.out.dump(sys.stdout) - result = result and run.success + decorate_summary(run2.summary.out, file, seed + count, run.buggify_enabled) + run2.summary.out.dump(sys.stdout) + result = result and run2.success count += 1 return result diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 4db6ae154a..4882f90b1b 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -28,18 +28,22 @@ class SummaryTree: self.children.append(element) def to_dict(self) -> Dict[str, Any]: - res: Dict[str, Any] = {'Type': self.name} - for k, v in self.attributes.items(): - res[k] = v children = [] for child in self.children: children.append(child.to_dict()) + if len(children) > 0 and len(self.attributes) == 0: + return {self.name: children} + res: Dict[str, Any] = {'Type': self.name} + for k, v in self.attributes.items(): + res[k] = v if len(children) > 0: res['children'] = children return res - def to_json(self, out: TextIO): - json.dump(self.to_dict(), out, indent=(' ' if config.pretty_print else None)) + def to_json(self, out: TextIO, prefix: str = ''): + res = json.dumps(self.to_dict(), indent=(' ' if config.pretty_print else None)) + for line in res.splitlines(False): + out.write('{}{}\n'.format(prefix, line)) def to_xml(self, out: TextIO, prefix: str = ''): # minidom doesn't support omitting the xml declaration which is a problem for joshua @@ -79,14 +83,15 @@ class SummaryTree: out.write('\n') child.to_xml(out, prefix=(' {}'.format(prefix) if config.pretty_print else prefix)) if len(self.children) > 0: - out.write('{}'.format(('\n' if config.pretty_print else ''), self.name)) + out.write('{}{}'.format(('\n' if config.pretty_print else ''), prefix, self.name)) - def dump(self, out: TextIO): + def dump(self, out: TextIO, prefix: str = '', new_line: bool = True): if config.output_format == 'json': - self.to_json(out) + self.to_json(out, prefix=prefix) else: - self.to_xml(out) - out.write('\n') + self.to_xml(out, prefix=prefix) + if new_line: + out.write('\n') ParserCallback = Callable[[Dict[str, str]], Optional[str]] From 6fc91096a8470e63e230cb9ce0fa8f3e55ab753b Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sun, 21 Aug 2022 11:13:39 -0600 Subject: [PATCH 10/29] generate reproduction command in result --- contrib/TestHarness2/test_harness/config.py | 3 +++ contrib/TestHarness2/test_harness/joshua.py | 26 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index b4b4b34113..e90464f87b 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -141,6 +141,9 @@ class Config: self.max_stderr_bytes: int = 1000 self.write_stats: bool = True self.read_stats: bool = True + self.reproduce_prefix: str | None = None + self.reproduce_prefix_args = {'type': str, 'required': False, + 'help': 'When printing the results, prepend this string to the command'} self._env_names: Dict[str, str] = {} self._config_map = self._build_map() self._read_env() diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 6411bb6cc3..b1b1d575e0 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -3,9 +3,12 @@ from __future__ import annotations import sys import xml.sax import xml.sax.handler +from pathlib import Path from typing import List from joshua import joshua_model + +import test_harness.run from test_harness.config import config from test_harness.summarize import SummaryTree @@ -60,4 +63,27 @@ def print_errors(ensemble_id: str): summary = ToSummaryTree() xml.sax.parseString(output, summary) res = summary.result() + cmd = [] + if config.reproduce_prefix is not None: + cmd.append(config.reproduce_prefix) + cmd.append('fdbserver') + if 'TestFile' in res.attributes: + file_name = res.attributes['TestFile'] + role = 'test' if test_harness.run.is_no_sim(Path(file_name)) else 'simulation' + cmd += ['-r', role, '-f', file_name] + else: + cmd += ['-r', 'simulation', '-f', ''] + if 'BuggifyEnabled' in res.attributes: + arg = 'on' + if res.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']: + arg = 'off' + cmd += ['-b', arg] + else: + cmd += ['b', ''] + cmd += ['--crash', '--trace_format', 'json'] + # we want the command as the first attribute + attributes = {'Command': ' '.join(cmd)} + for k, v in res.attributes: + attributes[k] = v + res.attributes = attributes res.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) From 951f426832bde2657bac0b205d1b6cc04e95635e Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sun, 21 Aug 2022 11:38:28 -0600 Subject: [PATCH 11/29] Fix unseed check --- contrib/TestHarness2/test_harness/run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index ce05dbf8b1..3eb52b397a 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -293,7 +293,7 @@ class ResourceMonitor(threading.Thread): class TestRun: def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID, - restarting: bool = False, test_determinism: bool = False, + restarting: bool = False, test_determinism: bool = False, buggify_enabled: bool = False, stats: str | None = None, expected_unseed: int | None = None, will_restart: bool = False): self.binary = binary self.test_file = test_file @@ -306,7 +306,7 @@ class TestRun: self.err_out: str = 'error.xml' self.use_valgrind: bool = config.use_valgrind self.old_binary_path: Path = config.old_binaries_path - self.buggify_enabled: bool = config.random.random() < config.buggify_on_ratio + self.buggify_enabled: bool = buggify_enabled self.fault_injection_enabled: bool = True self.trace_format: str | None = config.trace_format if Version.of_binary(self.binary) < "6.1.0": @@ -428,11 +428,12 @@ class TestRunner: will_restart = count + 1 < len(test_files) binary = self.binary_chooser.choose_binary(file) unseed_check = config.random.random() < config.unseed_check_ratio + buggify_enabled: bool = config.random.random() < config.buggify_on_ratio if unseed_check and count != 0: # for restarting tests we will need to restore the sim2 after the first run self.backup_sim_dir(seed + count - 1) run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, - stats=test_picker.dump_stats(), will_restart=will_restart) + stats=test_picker.dump_stats(), will_restart=will_restart, buggify_enabled=buggify_enabled) result = result and run.success test_picker.add_time(test_files[0], run.run_time, run.summary.out) decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled) @@ -444,11 +445,13 @@ class TestRunner: self.restore_sim_dir(seed + count - 1) run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, - will_restart=will_restart) + will_restart=will_restart, buggify_enabled=buggify_enabled) test_picker.add_time(file, run2.run_time, run.summary.out) decorate_summary(run2.summary.out, file, seed + count, run.buggify_enabled) run2.summary.out.dump(sys.stdout) result = result and run2.success + if not result: + return False count += 1 return result From 8d4d09c470a2fe557df909c00be5aef74d8821e5 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Sun, 21 Aug 2022 11:41:05 -0600 Subject: [PATCH 12/29] fix attribute copying --- contrib/TestHarness2/test_harness/joshua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index b1b1d575e0..93eb6786a1 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -83,7 +83,7 @@ def print_errors(ensemble_id: str): cmd += ['--crash', '--trace_format', 'json'] # we want the command as the first attribute attributes = {'Command': ' '.join(cmd)} - for k, v in res.attributes: + for k, v in res.attributes.items(): attributes[k] = v res.attributes = attributes res.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) From 9781ecb1f7f0185fa98da0244de62bb17f37bbba Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 09:57:35 -0600 Subject: [PATCH 13/29] fixed compiler warning --- fdbserver/include/fdbserver/DataDistribution.actor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/include/fdbserver/DataDistribution.actor.h b/fdbserver/include/fdbserver/DataDistribution.actor.h index 543648c6f5..c508d84624 100644 --- a/fdbserver/include/fdbserver/DataDistribution.actor.h +++ b/fdbserver/include/fdbserver/DataDistribution.actor.h @@ -352,7 +352,7 @@ FDB_DECLARE_BOOLEAN_PARAM(MoveKeyRangeOutPhysicalShard); class PhysicalShardCollection : public ReferenceCounted { public: - PhysicalShardCollection() : requireTransition(false), lastTransitionStartTime(now()) {} + PhysicalShardCollection() : lastTransitionStartTime(now()), requireTransition(false) {} enum class PhysicalShardCreationTime { DDInit, DDRelocator }; From 180024b76d6cbe648b2dca57eb904e39b04528d4 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 09:57:44 -0600 Subject: [PATCH 14/29] implemented testClass and testPriority --- contrib/TestHarness2/test_harness/fdb.py | 27 ++++++++++++++++--- contrib/TestHarness2/test_harness/results.py | 7 ++--- .../TestHarness2/test_harness/summarize.py | 1 + fdbserver/SimulatedCluster.actor.cpp | 12 +++++++-- fdbserver/tester.actor.cpp | 3 +++ flow/include/flow/Trace.h | 1 + tests/fast/BackupAzureBlobCorrectness.toml | 2 ++ tests/fast/BackupCorrectness.toml | 2 ++ tests/fast/BackupCorrectnessClean.toml | 2 ++ tests/fast/BackupS3BlobCorrectness.toml | 2 ++ tests/fast/BackupToDBCorrectness.toml | 2 ++ tests/fast/BackupToDBCorrectnessClean.toml | 2 ++ tests/fast/BlobGranuleMoveVerifyCycle.toml | 1 + tests/fast/BlobGranuleVerifyAtomicOps.toml | 1 + tests/fast/BlobGranuleVerifyCycle.toml | 1 + tests/fast/BlobGranuleVerifySmall.toml | 1 + tests/fast/BlobGranuleVerifySmallClean.toml | 1 + tests/fast/ChangeFeedOperations.toml | 1 + tests/fast/ChangeFeedOperationsMove.toml | 1 + tests/fast/ChangeFeeds.toml | 1 + tests/fast/EncryptKeyProxyTest.toml | 3 +++ tests/fast/EncryptionOps.toml | 3 +++ .../from_7.0.0/UpgradeAndBackupRestore-1.toml | 1 + .../from_7.0.0/UpgradeAndBackupRestore-2.toml | 2 ++ .../from_7.1.0/SnapCycleRestart-1.txt | 1 + .../from_7.1.0/SnapCycleRestart-2.txt | 1 + .../from_7.1.0/SnapIncrementalRestore-1.txt | 1 + .../from_7.1.0/SnapIncrementalRestore-2.txt | 1 + .../from_7.1.0/SnapTestAttrition-1.txt | 1 + .../from_7.1.0/SnapTestAttrition-2.txt | 1 + .../from_7.1.0/SnapTestRestart-1.txt | 1 + .../from_7.1.0/SnapTestRestart-2.txt | 1 + .../from_7.1.0/SnapTestSimpleRestart-1.txt | 1 + .../from_7.1.0/SnapTestSimpleRestart-2.txt | 1 + 34 files changed, 81 insertions(+), 9 deletions(-) diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index 0eed36bae3..2cfa9a2f05 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -4,6 +4,7 @@ from typing import OrderedDict, Tuple, List import collections import fdb +import fdb.tuple import struct from test_harness.run import StatFetcher, TestDescription @@ -29,6 +30,7 @@ def open_db(cluster_file: str | None): fdb_db = fdb.open(cluster_file) return fdb_db + def chunkify(iterable, sz: int): count = 0 res = [] @@ -44,17 +46,34 @@ def chunkify(iterable, sz: int): @fdb.transactional -def write_coverage_chunk(tr, path: Tuple[str, ...], coverage: List[Tuple[Coverage, bool]]): +def write_coverage_chunk(tr, path: Tuple[str, ...], metadata: Tuple[str, ...], + coverage: List[Tuple[Coverage, bool]], initialized: bool) -> bool: cov_dir = fdb.directory.create_or_open(tr, path) + if not initialized: + metadata_dir = fdb.directory.create_or_open(tr, metadata) + v = tr[metadata_dir['initialized']] + initialized = v.present() for cov, covered in coverage: - tr.add(cov_dir.pack((cov.file, cov.line, cov.comment)), struct.pack(' self.ratio + else: + self.coverage_ok = False def dump(self, prefix: str): errors = 0 @@ -70,7 +71,7 @@ class EnsembleResults: child.attributes['Severity'] = str(severity) child.attributes['File'] = cov.file child.attributes['Line'] = str(cov.line) - child.attributes['Comment'] = cov.comment + child.attributes['Comment'] = '' if cov.comment is None else cov.comment child.attributes['HitCount'] = str(count) out.append(child) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 4882f90b1b..805e566d63 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -396,6 +396,7 @@ class Summary: import test_harness.fdb test_harness.fdb.write_coverage(config.cluster_file, test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage',), + test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage-metadata',), self.coverage) def ok(self): diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 0ee75fd823..9c73515e81 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -84,7 +84,7 @@ bool destructed = false; class TestConfig { class ConfigBuilder { using value_type = toml::basic_value; - using base_variant = std::variant, ConfigDBType>; + using base_variant = std::variant, ConfigDBType>; using types = variant_map>, std::add_pointer_t>; std::unordered_map confMap; @@ -94,6 +94,10 @@ class TestConfig { visitor(const value_type& v) : value(v) {} void operator()(int* val) const { *val = value.as_integer(); } void operator()(Optional* val) const { *val = value.as_integer(); } + void operator()(float* val) const { *val = value.as_floating(); } + void operator()(Optional* val) const { *val = value.as_floating(); } + void operator()(double* val) const { *val = value.as_floating(); } + void operator()(Optional* val) const { *val = value.as_floating(); } void operator()(bool* val) const { *val = value.as_boolean(); } void operator()(Optional* val) const { *val = value.as_boolean(); } void operator()(std::string* val) const { *val = value.as_string(); } @@ -344,6 +348,8 @@ public: bool allowCreatingTenants = true; bool injectTargetedSSRestart = false; bool injectSSDelay = false; + std::string testClass; // unused -- used in TestHarness + float testPriority; // unused -- used in TestHarness ConfigDBType getConfigDBType() const { return configDBType; } @@ -371,7 +377,9 @@ public: } std::string extraDatabaseModeStr; ConfigBuilder builder; - builder.add("extraDatabaseMode", &extraDatabaseModeStr) + builder.add("testClass", &testClass) + .add("testPriority", &testPriority) + .add("extraDatabaseMode", &extraDatabaseModeStr) .add("extraDatabaseCount", &extraDatabaseCount) .add("minimumReplication", &minimumReplication) .add("minimumRegions", &minimumRegions) diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 723986a3cf..86616ee5d8 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1146,6 +1146,9 @@ ACTOR Future runTest(Database cx, std::map> testSpecGlobalKeys = { // These are read by SimulatedCluster and used before testers exist. Thus, they must // be recognized and accepted, but there's no point in placing them into a testSpec. + // testClass and testPriority are only used for TestHarness, we'll ignore those here + { "testClass", [](std::string const&) {} }, + { "testPriority", [](std::string const&) {} }, { "extraDatabaseMode", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } }, { "extraDatabaseCount", diff --git a/flow/include/flow/Trace.h b/flow/include/flow/Trace.h index a0b8d73b4d..6b23e8592e 100644 --- a/flow/include/flow/Trace.h +++ b/flow/include/flow/Trace.h @@ -256,6 +256,7 @@ FORMAT_TRACEABLE(long int, "%ld"); FORMAT_TRACEABLE(unsigned long int, "%lu"); FORMAT_TRACEABLE(long long int, "%lld"); FORMAT_TRACEABLE(unsigned long long int, "%llu"); +FORMAT_TRACEABLE(float, "%g"); FORMAT_TRACEABLE(double, "%g"); FORMAT_TRACEABLE(void*, "%p"); FORMAT_TRACEABLE(volatile long, "%ld"); diff --git a/tests/fast/BackupAzureBlobCorrectness.toml b/tests/fast/BackupAzureBlobCorrectness.toml index febf4eb9f8..2401f395df 100644 --- a/tests/fast/BackupAzureBlobCorrectness.toml +++ b/tests/fast/BackupAzureBlobCorrectness.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [[test]] testTitle = 'Cycle' clearAfterTest = 'false' diff --git a/tests/fast/BackupCorrectness.toml b/tests/fast/BackupCorrectness.toml index 1315b1af5f..0582bfa0eb 100644 --- a/tests/fast/BackupCorrectness.toml +++ b/tests/fast/BackupCorrectness.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [[test]] testTitle = 'BackupAndRestore' clearAfterTest = false diff --git a/tests/fast/BackupCorrectnessClean.toml b/tests/fast/BackupCorrectnessClean.toml index d5fc3d945e..d581e0caea 100644 --- a/tests/fast/BackupCorrectnessClean.toml +++ b/tests/fast/BackupCorrectnessClean.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [[test]] testTitle = 'BackupAndRestore' clearAfterTest = false diff --git a/tests/fast/BackupS3BlobCorrectness.toml b/tests/fast/BackupS3BlobCorrectness.toml index e80fa847b0..d642cd4811 100644 --- a/tests/fast/BackupS3BlobCorrectness.toml +++ b/tests/fast/BackupS3BlobCorrectness.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [[test]] testTitle = 'Cycle' clearAfterTest = 'false' diff --git a/tests/fast/BackupToDBCorrectness.toml b/tests/fast/BackupToDBCorrectness.toml index a5e80590c4..b8c9e859b8 100644 --- a/tests/fast/BackupToDBCorrectness.toml +++ b/tests/fast/BackupToDBCorrectness.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [configuration] extraDatabaseMode = 'LocalOrSingle' diff --git a/tests/fast/BackupToDBCorrectnessClean.toml b/tests/fast/BackupToDBCorrectnessClean.toml index 4ef236bc01..e6a9921383 100644 --- a/tests/fast/BackupToDBCorrectnessClean.toml +++ b/tests/fast/BackupToDBCorrectnessClean.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [configuration] extraDatabaseMode = 'LocalOrSingle' diff --git a/tests/fast/BlobGranuleMoveVerifyCycle.toml b/tests/fast/BlobGranuleMoveVerifyCycle.toml index c5d2d130ed..4f1267aac7 100644 --- a/tests/fast/BlobGranuleMoveVerifyCycle.toml +++ b/tests/fast/BlobGranuleMoveVerifyCycle.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "BlobGranule" blobGranulesEnabled = true allowDefaultTenant = false # FIXME: re-enable rocks at some point diff --git a/tests/fast/BlobGranuleVerifyAtomicOps.toml b/tests/fast/BlobGranuleVerifyAtomicOps.toml index 6cf5612898..84e946c004 100644 --- a/tests/fast/BlobGranuleVerifyAtomicOps.toml +++ b/tests/fast/BlobGranuleVerifyAtomicOps.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "BlobGranule" blobGranulesEnabled = true allowDefaultTenant = false injectTargetedSSRestart = true diff --git a/tests/fast/BlobGranuleVerifyCycle.toml b/tests/fast/BlobGranuleVerifyCycle.toml index 436d5f5cfe..1cf056b87b 100644 --- a/tests/fast/BlobGranuleVerifyCycle.toml +++ b/tests/fast/BlobGranuleVerifyCycle.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "BlobGranule" blobGranulesEnabled = true allowDefaultTenant = false injectTargetedSSRestart = true diff --git a/tests/fast/BlobGranuleVerifySmall.toml b/tests/fast/BlobGranuleVerifySmall.toml index 97ced2772c..ba50b0fda6 100644 --- a/tests/fast/BlobGranuleVerifySmall.toml +++ b/tests/fast/BlobGranuleVerifySmall.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "BlobGranule" blobGranulesEnabled = true allowDefaultTenant = false injectTargetedSSRestart = true diff --git a/tests/fast/BlobGranuleVerifySmallClean.toml b/tests/fast/BlobGranuleVerifySmallClean.toml index 6630fba126..ef957b9f53 100644 --- a/tests/fast/BlobGranuleVerifySmallClean.toml +++ b/tests/fast/BlobGranuleVerifySmallClean.toml @@ -3,6 +3,7 @@ blobGranulesEnabled = true allowDefaultTenant = false # FIXME: re-enable rocks at some point storageEngineExcludeTypes = [4, 5] +testClass = "BlobGranule" [[test]] testTitle = 'BlobGranuleVerifySmallClean' diff --git a/tests/fast/ChangeFeedOperations.toml b/tests/fast/ChangeFeedOperations.toml index ca3d47e08e..53f3bb8b19 100644 --- a/tests/fast/ChangeFeedOperations.toml +++ b/tests/fast/ChangeFeedOperations.toml @@ -1,5 +1,6 @@ [configuration] allowDefaultTenant = false +testClass = "ChangeFeeds" # TODO add failure events, and then add a version that also supports randomMoveKeys diff --git a/tests/fast/ChangeFeedOperationsMove.toml b/tests/fast/ChangeFeedOperationsMove.toml index a9852f2760..53808227b4 100644 --- a/tests/fast/ChangeFeedOperationsMove.toml +++ b/tests/fast/ChangeFeedOperationsMove.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "ChangeFeeds" allowDefaultTenant = false # TODO add failure events, and then add a version that also supports randomMoveKeys diff --git a/tests/fast/ChangeFeeds.toml b/tests/fast/ChangeFeeds.toml index 8f2d348dd3..e51e8c7c7a 100644 --- a/tests/fast/ChangeFeeds.toml +++ b/tests/fast/ChangeFeeds.toml @@ -1,4 +1,5 @@ [configuration] +testClass = "ChangeFeeds" allowDefaultTenant = false [[test]] diff --git a/tests/fast/EncryptKeyProxyTest.toml b/tests/fast/EncryptKeyProxyTest.toml index 3f5f7e12f1..199cc72da6 100644 --- a/tests/fast/EncryptKeyProxyTest.toml +++ b/tests/fast/EncryptKeyProxyTest.toml @@ -1,3 +1,6 @@ +[configuration] +testClass = "Encryption" + [[knobs]] enable_encryption = true diff --git a/tests/fast/EncryptionOps.toml b/tests/fast/EncryptionOps.toml index f3150dc0ae..9aaa09f177 100644 --- a/tests/fast/EncryptionOps.toml +++ b/tests/fast/EncryptionOps.toml @@ -1,3 +1,6 @@ +[configuration] +testClass = "Encryption" + [[knobs]] enable_encryption = false diff --git a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml index f0f2141778..04144efd6b 100644 --- a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml +++ b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml @@ -1,3 +1,4 @@ +testClass = "Backup" storageEngineExcludeTypes=3 [[test]] diff --git a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml index b5ac855d52..22f016deab 100644 --- a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml +++ b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml @@ -1,3 +1,5 @@ +testClass = "Backup" + [[test]] testTitle = 'SecondCycleTest' simBackupAgents = 'BackupToFile' diff --git a/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt b/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt index 2df70a8f0f..84eadec755 100644 --- a/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt +++ b/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[3, 4, 5] ;Take snap and do cycle test diff --git a/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt b/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt index 9ec734929e..c0366ae2f4 100644 --- a/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt +++ b/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[4, 5] buggify=off diff --git a/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt b/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt index b464eda8e3..22bd11c01f 100644 --- a/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt +++ b/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[3, 4, 5] logAntiQuorum = 0 diff --git a/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt b/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt index 1aa1596fc7..fc6034373a 100644 --- a/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt +++ b/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[4, 5] testTitle=RestoreBackup diff --git a/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt b/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt index 84ff6bccbc..826d756e88 100644 --- a/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt +++ b/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[3, 4, 5] ;write 1000 Keys ending with even numbers diff --git a/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt b/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt index dde9dab0e2..8a62641c1e 100644 --- a/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt +++ b/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[4, 5] buggify=off diff --git a/tests/restarting/from_7.1.0/SnapTestRestart-1.txt b/tests/restarting/from_7.1.0/SnapTestRestart-1.txt index e47db390e1..d501cad91a 100644 --- a/tests/restarting/from_7.1.0/SnapTestRestart-1.txt +++ b/tests/restarting/from_7.1.0/SnapTestRestart-1.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[3, 4, 5] ;write 1000 Keys ending with even numbers diff --git a/tests/restarting/from_7.1.0/SnapTestRestart-2.txt b/tests/restarting/from_7.1.0/SnapTestRestart-2.txt index 8ee3b0b5ab..2c67691278 100644 --- a/tests/restarting/from_7.1.0/SnapTestRestart-2.txt +++ b/tests/restarting/from_7.1.0/SnapTestRestart-2.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[4, 5] buggify=off diff --git a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt index bb5a85efe1..82532d4c90 100644 --- a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt +++ b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[3, 4, 5] ;write 1000 Keys ending with even number diff --git a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt index 103eb53d1d..2e03bf7eac 100644 --- a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt +++ b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt @@ -1,3 +1,4 @@ +testClass=SnapshotTest storageEngineExcludeTypes=[4, 5] buggify=off From 798ae8913c95137bf65e7607c0f756abb50be3db Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 10:34:24 -0600 Subject: [PATCH 15/29] fixed detailed output and move success to own flag --- contrib/TestHarness2/test_harness/config.py | 4 +- contrib/TestHarness2/test_harness/joshua.py | 61 +++++++++++---------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index e90464f87b..92510618b0 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -133,7 +133,9 @@ class Config: self.exclude_test_names: str = r'.^' self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'} self.details: bool = False - self.details_args = {'help': 'Print detailed results', 'short_name': 'c'} + self.details_args = {'help': 'Print detailed results', 'short_name': 'c', 'action': 'store_true'} + self.success: bool = False + self.success_args = {'help': 'Print successful results', 'action': 'store_true'} self.cov_include_files: str = r'.*' self.cov_include_files_args = {'help': 'Only consider coverage traces that originated in files matching regex'} self.cov_exclude_files: str = r'.^' diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 93eb6786a1..53d5f9d279 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -38,11 +38,38 @@ class ToSummaryTree(xml.sax.handler.ContentHandler): self.stack[-1].children.append(closed) +def _print_summary(summary: SummaryTree): + cmd = [] + if config.reproduce_prefix is not None: + cmd.append(config.reproduce_prefix) + cmd.append('fdbserver') + if 'TestFile' in summary.attributes: + file_name = summary.attributes['TestFile'] + role = 'test' if test_harness.run.is_no_sim(Path(file_name)) else 'simulation' + cmd += ['-r', role, '-f', file_name] + else: + cmd += ['-r', 'simulation', '-f', ''] + if 'BuggifyEnabled' in summary.attributes: + arg = 'on' + if summary.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']: + arg = 'off' + cmd += ['-b', arg] + else: + cmd += ['b', ''] + cmd += ['--crash', '--trace_format', 'json'] + # we want the command as the first attribute + attributes = {'Command': ' '.join(cmd)} + for k, v in summary.attributes.items(): + attributes[k] = v + summary.attributes = attributes + summary.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) + + def print_errors(ensemble_id: str): joshua_model.open(config.cluster_file) properties = joshua_model.get_ensemble_properties(ensemble_id) compressed = properties["compressed"] if "compressed" in properties else False - for rec in joshua_model.tail_results(ensemble_id, errors_only=(not config.details), compressed=compressed): + for rec in joshua_model.tail_results(ensemble_id, errors_only=(not config.success), compressed=compressed): if len(rec) == 5: version_stamp, result_code, host, seed, output = rec elif len(rec) == 4: @@ -60,30 +87,8 @@ def print_errors(ensemble_id: str): seed = None else: raise Exception("Unknown result format") - summary = ToSummaryTree() - xml.sax.parseString(output, summary) - res = summary.result() - cmd = [] - if config.reproduce_prefix is not None: - cmd.append(config.reproduce_prefix) - cmd.append('fdbserver') - if 'TestFile' in res.attributes: - file_name = res.attributes['TestFile'] - role = 'test' if test_harness.run.is_no_sim(Path(file_name)) else 'simulation' - cmd += ['-r', role, '-f', file_name] - else: - cmd += ['-r', 'simulation', '-f', ''] - if 'BuggifyEnabled' in res.attributes: - arg = 'on' - if res.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']: - arg = 'off' - cmd += ['-b', arg] - else: - cmd += ['b', ''] - cmd += ['--crash', '--trace_format', 'json'] - # we want the command as the first attribute - attributes = {'Command': ' '.join(cmd)} - for k, v in res.attributes.items(): - attributes[k] = v - res.attributes = attributes - res.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) + lines = output.splitlines() + for line in lines: + summary = ToSummaryTree() + xml.sax.parseString(line, summary) + _print_summary(summary.result()) From 41ac372931a5641b742c39b08c7cec6c908ea830 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 20:38:42 -0600 Subject: [PATCH 16/29] test_harness.result generates valid json --- contrib/TestHarness2/test_harness/config.py | 1 + contrib/TestHarness2/test_harness/joshua.py | 80 +++++++++++++++++-- contrib/TestHarness2/test_harness/results.py | 35 +++++--- .../TestHarness2/test_harness/summarize.py | 31 +++++-- 4 files changed, 123 insertions(+), 24 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 92510618b0..0e205cf1db 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -70,6 +70,7 @@ class ConfigValue: return self.name, args.__getattribute__(self.get_arg_name()) +# TODO: document this class class Config: def __init__(self): self.random = random.Random() diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 53d5f9d279..f4215918a7 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -1,10 +1,12 @@ from __future__ import annotations +import collections +import io import sys import xml.sax import xml.sax.handler from pathlib import Path -from typing import List +from typing import List, OrderedDict, Set from joshua import joshua_model @@ -38,7 +40,7 @@ class ToSummaryTree(xml.sax.handler.ContentHandler): self.stack[-1].children.append(closed) -def _print_summary(summary: SummaryTree): +def _print_summary(summary: SummaryTree, commands: Set[str]) -> str: cmd = [] if config.reproduce_prefix is not None: cmd.append(config.reproduce_prefix) @@ -49,6 +51,10 @@ def _print_summary(summary: SummaryTree): cmd += ['-r', role, '-f', file_name] else: cmd += ['-r', 'simulation', '-f', ''] + if 'RandomSeed' in summary.attributes: + cmd += ['-s', summary.attributes['RandomSeed']] + else: + cmd += ['-s', ''] if 'BuggifyEnabled' in summary.attributes: arg = 'on' if summary.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']: @@ -57,12 +63,75 @@ def _print_summary(summary: SummaryTree): else: cmd += ['b', ''] cmd += ['--crash', '--trace_format', 'json'] + key = ' '.join(cmd) + count = 1 + while key in commands: + key = '{} # {}'.format(' '.join(cmd), count) + count += 1 # we want the command as the first attribute attributes = {'Command': ' '.join(cmd)} for k, v in summary.attributes.items(): - attributes[k] = v + if k == 'Errors': + attributes['ErrorCount'] = v + else: + attributes[k] = v summary.attributes = attributes - summary.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) + if config.details: + key = str(len(commands)) + str_io = io.StringIO() + summary.dump(str_io, prefix=(' ' if config.pretty_print else '')) + if config.output_format == 'json': + sys.stdout.write('{}"Test{}": {}'.format(' ' if config.pretty_print else '', + key, str_io.getvalue())) + else: + sys.stdout.write(str_io.getvalue()) + if config.pretty_print: + sys.stdout.write('\n' if config.output_format == 'xml' else ',\n') + return key + error_count = 0 + warning_count = 0 + small_summary = SummaryTree('Test') + small_summary.attributes = attributes + errors = SummaryTree('Errors') + warnings = SummaryTree('Warnings') + buggifies: OrderedDict[str, List[int]] = collections.OrderedDict() + for child in summary.children: + if 'Severity' in child.attributes and child.attributes['Severity'] == '40' and error_count < config.max_errors: + error_count += 1 + errors.append(child) + if 'Severity' in child.attributes and child.attributes[ + 'Severity'] == '30' and warning_count < config.max_warnings: + warning_count += 1 + warnings.append(child) + if child.name == 'BuggifySection': + file = child.attributes['File'] + line = int(child.attributes['Line']) + if file in buggifies: + buggifies[file].append(line) + else: + buggifies[file] = [line] + buggifies_elem = SummaryTree('Buggifies') + for file, lines in buggifies.items(): + lines.sort() + if config.output_format == 'json': + buggifies_elem.attributes[file] = ' '.join(str(line) for line in lines) + else: + child = SummaryTree('Buggify') + child.attributes['File'] = file + child.attributes['Lines'] = ' '.join(str(line) for line in lines) + small_summary.append(child) + small_summary.children.append(buggifies_elem) + if len(errors.children) > 0: + small_summary.children.append(errors) + if len(warnings.children) > 0: + small_summary.children.append(warnings) + output = io.StringIO() + small_summary.dump(output, prefix=(' ' if config.pretty_print else '')) + if config.output_format == 'json': + sys.stdout.write('{}"{}": {}'.format(' ' if config.pretty_print else '', key, output.getvalue().strip())) + else: + sys.stdout.write(output.getvalue().strip()) + sys.stdout.write('\n' if config.output_format == 'xml' else ',\n') def print_errors(ensemble_id: str): @@ -88,7 +157,8 @@ def print_errors(ensemble_id: str): else: raise Exception("Unknown result format") lines = output.splitlines() + commands: Set[str] = set() for line in lines: summary = ToSummaryTree() xml.sax.parseString(line, summary) - _print_summary(summary.result()) + commands.add(_print_summary(summary.result(), commands)) diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index a2695bbe31..e3292db822 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -1,14 +1,16 @@ from __future__ import annotations +import argparse +import io +import json import re import sys -from typing import List, Tuple, OrderedDict +import test_harness.fdb +from typing import List, Tuple, OrderedDict from test_harness.summarize import SummaryTree, Coverage from test_harness.config import config - -import argparse -import test_harness.fdb +from xml.sax.saxutils import quoteattr class GlobalStatistics: @@ -84,18 +86,25 @@ class EnsembleResults: out.append(child) if errors > 0: out.attributes['Errors'] = str(errors) - out.dump(sys.stdout, prefix=prefix, new_line=config.pretty_print) + str_io = io.StringIO() + out.dump(str_io, prefix=prefix, new_line=config.pretty_print) + if config.output_format == 'xml': + sys.stdout.write(str_io.getvalue()) + else: + sys.stdout.write('{}"EnsembleResults":{}{}'.format(' ' if config.pretty_print else '', + '\n' if config.pretty_print else ' ', + str_io.getvalue())) def write_header(ensemble_id: str): if config.output_format == 'json': if config.pretty_print: print('{') - print(' {}: {}'.format('ID', ensemble_id)) + print(' "{}": {},\n'.format('ID', json.dumps(ensemble_id.strip()))) else: - sys.stdout.write('{{{}: {}'.format('ID', ensemble_id)) + sys.stdout.write('{{{}: {},'.format('ID', json.dumps(ensemble_id.strip()))) elif config.output_format == 'xml': - sys.stdout.write(''.format(ensemble_id)) + sys.stdout.write(''.format(quoteattr(ensemble_id.strip()))) if config.pretty_print: sys.stdout.write('\n') else: @@ -103,10 +112,10 @@ def write_header(ensemble_id: str): def write_footer(): - if config.output_format == 'json': - sys.stdout.write('}') - elif config.output_format == 'xml': - sys.stdout.write('') + if config.output_format == 'xml': + sys.stdout.write('\n') + elif config.output_format == 'json': + sys.stdout.write('}\n') else: assert False, 'unknown output format {}'.format(config.output_format) @@ -114,6 +123,7 @@ def write_footer(): if __name__ == '__main__': parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.change_default('pretty_print', True) + config.change_default('max_warnings', 0) config.build_arguments(parser) parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for') args = parser.parse_args() @@ -130,4 +140,5 @@ if __name__ == '__main__': child.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) results = EnsembleResults(config.cluster_file, args.ensemble_id) results.dump(' ' if config.pretty_print else '') + write_footer() exit(0 if results.coverage_ok else 1) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 805e566d63..efad73d88a 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -27,15 +27,32 @@ class SummaryTree: def append(self, element: SummaryTree): self.children.append(element) - def to_dict(self) -> Dict[str, Any]: - children = [] - for child in self.children: - children.append(child.to_dict()) - if len(children) > 0 and len(self.attributes) == 0: - return {self.name: children} - res: Dict[str, Any] = {'Type': self.name} + def to_dict(self, add_name: bool = True) -> Dict[str, Any] | [Any]: + if len(self.children) > 0 and len(self.attributes) == 0: + children = [] + for child in self.children: + children.append(child.to_dict()) + if add_name: + return {self.name: children} + else: + return children + res: Dict[str, Any] = {} + if add_name: + res['Type'] = self.name for k, v in self.attributes.items(): res[k] = v + children = [] + child_keys: Dict[str, int] = {} + for child in self.children: + if child.name in child_keys: + child_keys[child.name] += 1 + else: + child_keys[child.name] = 1 + for child in self.children: + if child_keys[child.name] == 1 and child.name not in self.attributes: + res[child.name] = child.to_dict(add_name=False) + else: + children.append(child.to_dict()) if len(children) > 0: res['children'] = children return res From 2d9f8ba3ea4c3f24f60cacbbe35e817b40ac2a0a Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 20:44:24 -0600 Subject: [PATCH 17/29] fixed xml formatting --- contrib/TestHarness2/test_harness/joshua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index f4215918a7..7b152d2c2c 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -130,7 +130,7 @@ def _print_summary(summary: SummaryTree, commands: Set[str]) -> str: if config.output_format == 'json': sys.stdout.write('{}"{}": {}'.format(' ' if config.pretty_print else '', key, output.getvalue().strip())) else: - sys.stdout.write(output.getvalue().strip()) + sys.stdout.write('{}{}'.format(' ' if config.pretty_print else '', output.getvalue().strip())) sys.stdout.write('\n' if config.output_format == 'xml' else ',\n') From 3545708b91603caf2fe4cd2453353e9939b2f600 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Mon, 22 Aug 2022 21:06:11 -0600 Subject: [PATCH 18/29] addressed review comments --- cmake/AddFdbTest.cmake | 1 - contrib/TestHarness2/test_harness/config.py | 36 +++++++++++++++++++-- contrib/TestHarness2/test_harness/fdb.py | 3 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 13fa8d6a23..16b1b66db6 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -229,7 +229,6 @@ function(stage_correctness_package) ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll - ${harness_files} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/CMakeCache.txt ${STAGE_OUT_DIR} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbserver ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 0e205cf1db..f41697c3d9 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -21,10 +21,14 @@ class BuggifyOption: self.value = BuggifyOptionValue.RANDOM if val is not None: v = val.lower() - if v == 'on' or v == '1' or v == 'true': + if v in ['on', '1', 'true']: self.value = BuggifyOptionValue.ON - elif v == 'off' or v == '0' or v == 'false': + elif v in ['off', '0', 'false']: self.value = BuggifyOptionValue.OFF + elif v in ['random', 'rnd', 'r']: + pass + else: + assert False, 'Invalid value {} -- use true, false, or random'.format(v) class ConfigValue: @@ -70,8 +74,34 @@ class ConfigValue: return self.name, args.__getattribute__(self.get_arg_name()) -# TODO: document this class class Config: + """ + This is the central configuration class for test harness. The values in this class are exposed globally through + a global variable test_harness.config.config. This class provides some "magic" to keep test harness flexible. + Each parameter can further be configured using an `_args` member variable which is expected to be a dictionary. + * The value of any variable can be set through the command line. For a variable named `variable_name` we will + by default create a new command line option `--variable-name` (`_` is automatically changed to `-`). This + default can be changed by setting the `'long_name'` property in the `_arg` dict. + * In addition the user can also optionally set a short-name. This can be achieved by setting the `'short_name'` + property in the `_arg` dictionary. + * All additional properties in `_args` are passed to `argparse.add_argument`. + * If the default of a variable is `None` the user should explicitly set the `'type'` property to an appropriate + type. + * In addition to command line flags, all configuration options can also be controlled through environment variables. + By default, `variable-name` can be changed by setting the environment variable `TH_VARIABLE_NAME`. This default + can be changed by setting the `'env_name'` property. + * Test harness comes with multiple executables. Each of these should use the config facility. For this, + `Config.build_arguments` should be called first with the `argparse` parser. Then `Config.extract_args` needs + to be called with the result of `argparse.ArgumentParser.parse_args`. A sample example could look like this: + ``` + parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + config.build_arguments(parser) + args = parser.parse_args() + config.extract_args(args) + ``` + * Changing the default value for all executables might not always be desirable. If it should be only changed for + one executable Config.change_default should be used. + """ def __init__(self): self.random = random.Random() self.cluster_file: str | None = None diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index 2cfa9a2f05..2e340b7f76 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -17,8 +17,7 @@ fdb.api_version(630) def str_to_tuple(s: str | None): if s is None: return s - res = s.split(',') - return tuple(res) + return tuple(s.split(',')) fdb_db = None From 43420a43efe707a529b3f822bfd99a432099e3d9 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 23 Aug 2022 14:38:58 -0600 Subject: [PATCH 19/29] port timeout scripts and use new test harness for valgrind --- contrib/Joshua/scripts/correctnessTimeout.sh | 6 +- contrib/Joshua/scripts/valgrindTest.sh | 2 +- contrib/Joshua/scripts/valgrindTimeout.sh | 6 +- contrib/TestHarness2/test_harness/run.py | 5 +- .../TestHarness2/test_harness/summarize.py | 10 +++- contrib/TestHarness2/test_harness/timeout.py | 60 +++++++++++++++++++ 6 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 contrib/TestHarness2/test_harness/timeout.py diff --git a/contrib/Joshua/scripts/correctnessTimeout.sh b/contrib/Joshua/scripts/correctnessTimeout.sh index 7917aae591..6bd0bfeee0 100755 --- a/contrib/Joshua/scripts/correctnessTimeout.sh +++ b/contrib/Joshua/scripts/correctnessTimeout.sh @@ -1,4 +1,4 @@ #!/bin/bash -u -for file in `find . -name 'trace*.xml'` ; do - mono ./bin/TestHarness.exe summarize "${file}" summary.xml "" JoshuaTimeout true -done + + +python3 -m test_harness.timeout diff --git a/contrib/Joshua/scripts/valgrindTest.sh b/contrib/Joshua/scripts/valgrindTest.sh index 5409429691..820750f3b2 100755 --- a/contrib/Joshua/scripts/valgrindTest.sh +++ b/contrib/Joshua/scripts/valgrindTest.sh @@ -1,3 +1,3 @@ #!/bin/sh OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}" -mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" true +python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR} --use-valgrind diff --git a/contrib/Joshua/scripts/valgrindTimeout.sh b/contrib/Joshua/scripts/valgrindTimeout.sh index b9d9e7ebad..2224598e43 100755 --- a/contrib/Joshua/scripts/valgrindTimeout.sh +++ b/contrib/Joshua/scripts/valgrindTimeout.sh @@ -1,6 +1,2 @@ #!/bin/bash -u -for file in `find . -name 'trace*.xml'` ; do - for valgrindFile in `find . -name 'valgrind*.xml'` ; do - mono ./bin/TestHarness.exe summarize "${file}" summary.xml "${valgrindFile}" JoshuaTimeout true - done -done +python3 -m test_harness.timeout --use-valgrind diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 3eb52b397a..f8e5aae3aa 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -303,7 +303,6 @@ class TestRun: self.test_determinism = test_determinism self.stats: str | None = stats self.expected_unseed: int | None = expected_unseed - self.err_out: str = 'error.xml' self.use_valgrind: bool = config.use_valgrind self.old_binary_path: Path = config.old_binaries_path self.buggify_enabled: bool = buggify_enabled @@ -334,11 +333,11 @@ class TestRun: valgrind_file: Path | None = None if self.use_valgrind: command.append('valgrind') - valgrind_file = Path('valgrind-{}.xml'.format(self.random_seed)) + valgrind_file = self.temp_path / Path('valgrind-{}.xml'.format(self.random_seed)) dbg_path = os.getenv('FDB_VALGRIND_DBGPATH') if dbg_path is not None: command.append('--extra-debuginfo-path={}'.format(dbg_path)) - command += ['--xml=yes', '--xml-file={}'.format(valgrind_file), '-q'] + command += ['--xml=yes', '--xml-file={}'.format(valgrind_file.absolute()), '-q'] command += [str(self.binary.absolute()), '-r', 'test' if is_no_sim(self.test_file) else 'simulation', '-f', str(self.test_file), diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index efad73d88a..73c5586bf2 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -395,6 +395,12 @@ class Summary: self.handler = ParseHandler(self.out) self.register_handlers() + def summarize_files(self, trace_files: List[Path]): + assert len(trace_files) > 0 + for f in trace_files: + self.parse_file(f) + self.done() + def summarize(self, trace_dir: Path, command: str): trace_files = TraceFiles(trace_dir) if len(trace_files) == 0: @@ -406,9 +412,7 @@ class Summary: child.attributes['Command'] = command self.out.append(child) return - for f in trace_files[0]: - self.parse_file(f) - self.done() + self.summarize_files(trace_files[0]) if config.joshua_dir is not None: import test_harness.fdb test_harness.fdb.write_coverage(config.cluster_file, diff --git a/contrib/TestHarness2/test_harness/timeout.py b/contrib/TestHarness2/test_harness/timeout.py new file mode 100644 index 0000000000..372d0795e7 --- /dev/null +++ b/contrib/TestHarness2/test_harness/timeout.py @@ -0,0 +1,60 @@ +import argparse +import re +import sys + +from pathlib import Path +from test_harness.config import config +from test_harness.summarize import Summary, TraceFiles +from typing import Pattern, List + + +def files_matching(path: Path, pattern: Pattern, recurs: bool = True) -> List[Path]: + res: List[Path] = [] + for file in path.iterdir(): + if file.is_file() and pattern.match(file.name) is not None: + res.append(file) + elif file.is_dir() and recurs: + res += files_matching(file, pattern, recurs) + return res + + +def dirs_with_files_matching(path: Path, pattern: Pattern, recurse: bool = True) -> List[Path]: + res: List[Path] = [] + sub_directories: List[Path] = [] + has_file = False + for file in path.iterdir(): + if file.is_file() and pattern.match(file.name) is not None: + has_file = True + elif file.is_dir() and recurse: + sub_directories.append(file) + if has_file: + res.append(path) + if recurse: + for file in sub_directories: + res += dirs_with_files_matching(file, pattern, recurse=True) + res.sort() + return res + + +if __name__ == '__main__': + parser = argparse.ArgumentParser('TestHarness Timeout', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + config.build_arguments(parser) + args = parser.parse_args() + config.extract_args(args) + valgrind_files: List[Path] = [] + if config.use_valgrind: + valgrind_files = files_matching(Path.cwd(), re.compile(r'valgrind.*\.xml')) + + for directory in dirs_with_files_matching(Path.cwd(), re.compile(r'trace.*\.(json|xml)'), recurse=True): + trace_files = TraceFiles(directory) + for files in trace_files: + if config.use_valgrind: + for valgrind_file in valgrind_files: + summary = Summary(Path('bin/fdbserver'), was_killed=True) + summary.valgrind_out_file = valgrind_file + summary.summarize_files(files) + summary.out.dump(sys.stdout) + else: + summary = Summary(Path('bin/fdbserver'), was_killed=True) + summary.summarize_files(files) + summary.out.dump(sys.stdout) From edad0f35d22696682fe3d3dc20af94631fd5a0c9 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 23 Aug 2022 18:06:11 -0600 Subject: [PATCH 20/29] valgrind reports include stack traces --- .../TestHarness2/test_harness/summarize.py | 80 ++-------- .../test_harness/test_valgrind_parser.py | 16 ++ contrib/TestHarness2/test_harness/valgrind.py | 141 ++++++++++++++++++ 3 files changed, 171 insertions(+), 66 deletions(-) create mode 100644 contrib/TestHarness2/test_harness/test_valgrind_parser.py create mode 100644 contrib/TestHarness2/test_harness/valgrind.py diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 73c5586bf2..df79cb95c3 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple from test_harness.config import config +from test_harness.valgrind import parse_valgrind_output class SummaryTree: @@ -216,69 +217,6 @@ def format_test_error(attrs: Dict[str, str], include_details: bool) -> str: return res -class ValgrindError: - def __init__(self, what: str = '', kind: str = ''): - self.what: str = what - self.kind: str = kind - - def __str__(self): - return 'ValgrindError(what="{}", kind="{}")'.format(self.what, self.kind) - - -class ValgrindHandler(xml.sax.handler.ContentHandler): - def __init__(self): - super().__init__() - self.stack: List[ValgrindError] = [] - self.result: List[ValgrindError] = [] - self.in_kind = False - self.in_what = False - - @staticmethod - def from_content(content): - if isinstance(content, bytes): - return content.decode() - assert isinstance(content, str) - return content - - def characters(self, content): - if len(self.stack) == 0: - return - elif self.in_kind: - self.stack[-1].kind += self.from_content(content) - elif self.in_what: - self.stack[-1].what += self.from_content(content) - - def startElement(self, name, attrs): - if name == 'error': - self.stack.append(ValgrindError()) - if len(self.stack) == 0: - return - if name == 'kind': - self.in_kind = True - elif name == 'what': - self.in_what = True - - def endElement(self, name): - if name == 'error': - self.result.append(self.stack.pop()) - elif name == 'kind': - self.in_kind = False - elif name == 'what': - self.in_what = False - - -def parse_valgrind_output(valgrind_out_file: Path) -> List[str]: - res: List[str] = [] - handler = ValgrindHandler() - with valgrind_out_file.open('r') as f: - xml.sax.parse(f, handler) - for err in handler.result: - if err.kind.startswith('Leak'): - continue - res.append(err.kind) - return res - - class Coverage: def __init__(self, file: str, line: str | int, comment: str | None = None): self.file = file @@ -454,18 +392,28 @@ class Summary: self.out.attributes['PeakMemory'] = str(self.max_rss) if self.valgrind_out_file is not None: try: - whats = parse_valgrind_output(self.valgrind_out_file) - for what in whats: + valgrind_errors = parse_valgrind_output(self.valgrind_out_file) + for valgrind_error in valgrind_errors: + if valgrind_error.kind.startswith('Leak'): + continue self.error = True child = SummaryTree('ValgrindError') child.attributes['Severity'] = '40' - child.attributes['What'] = what + child.attributes['What'] = valgrind_error.what.what + child.attributes['Backtrace'] = valgrind_error.what.backtrace + aux_count = 0 + for aux in valgrind_error.aux: + child.attributes['WhatAux{}'.format(aux_count)] = aux.what + child.attributes['BacktraceAux{}'.format(aux_count)] = aux.backtrace + aux_count += 1 self.out.append(child) except Exception as e: self.error = True child = SummaryTree('ValgrindParseError') child.attributes['Severity'] = '40' child.attributes['ErrorMessage'] = str(e) + _, _, exc_traceback = sys.exc_info() + child.attributes['Trace'] = repr(traceback.format_tb(exc_traceback)) self.out.append(child) self.error_list.append('Failed to parse valgrind output: {}'.format(str(e))) if not self.test_end_found: diff --git a/contrib/TestHarness2/test_harness/test_valgrind_parser.py b/contrib/TestHarness2/test_harness/test_valgrind_parser.py new file mode 100644 index 0000000000..0b36e8e6d5 --- /dev/null +++ b/contrib/TestHarness2/test_harness/test_valgrind_parser.py @@ -0,0 +1,16 @@ +import sys + +from test_harness.valgrind import parse_valgrind_output +from pathlib import Path + + +if __name__ == '__main__': + errors = parse_valgrind_output(Path(sys.argv[1])) + for valgrind_error in errors: + print('ValgrindError: what={}, kind={}'.format(valgrind_error.what.what, valgrind_error.kind)) + print('Backtrace: {}'.format(valgrind_error.what.backtrace)) + counter = 0 + for aux in valgrind_error.aux: + print('Aux {}:'.format(counter)) + print(' What: {}'.format(aux.what)) + print(' Backtrace: {}'.format(aux.backtrace)) diff --git a/contrib/TestHarness2/test_harness/valgrind.py b/contrib/TestHarness2/test_harness/valgrind.py new file mode 100644 index 0000000000..1be8af0d18 --- /dev/null +++ b/contrib/TestHarness2/test_harness/valgrind.py @@ -0,0 +1,141 @@ +import enum +import xml +import xml.sax.handler +from pathlib import Path +from typing import List + + +class ValgrindWhat: + def __init__(self): + self.what: str = '' + self.backtrace: str = '' + + +class ValgrindError: + def __init__(self): + self.what: ValgrindWhat = ValgrindWhat() + self.kind: str = '' + self.aux: List[ValgrindWhat] = [] + + +# noinspection PyArgumentList +class ValgrindParseState(enum.Enum): + ROOT = enum.auto() + ERROR = enum.auto() + ERROR_AUX = enum.auto() + KIND = enum.auto() + WHAT = enum.auto() + TRACE = enum.auto() + AUX_WHAT = enum.auto() + STACK = enum.auto() + STACK_AUX = enum.auto() + STACK_IP = enum.auto() + STACK_IP_AUX = enum.auto + + +class ValgrindHandler(xml.sax.handler.ContentHandler): + def __init__(self): + super().__init__() + self.stack: List[ValgrindError] = [] + self.result: List[ValgrindError] = [] + self.state_stack: List[ValgrindParseState] = [] + + def state(self) -> ValgrindParseState: + if len(self.state_stack) == 0: + return ValgrindParseState.ROOT + return self.state_stack[-1] + + @staticmethod + def from_content(content): + # pdb.set_trace() + if isinstance(content, bytes): + return content.decode() + assert isinstance(content, str) + return content + + def characters(self, content): + # pdb.set_trace() + state = self.state() + if len(self.state_stack) == 0: + return + else: + assert len(self.stack) > 0 + if state is ValgrindParseState.KIND: + self.stack[-1].kind += self.from_content(content) + elif state is ValgrindParseState.WHAT: + self.stack[-1].what.what += self.from_content(content) + elif state is ValgrindParseState.AUX_WHAT: + self.stack[-1].aux[-1].what += self.from_content(content) + elif state is ValgrindParseState.STACK_IP: + self.stack[-1].what.backtrace += self.from_content(content) + elif state is ValgrindParseState.STACK_IP_AUX: + self.stack[-1].aux[-1].backtrace += self.from_content(content) + + def startElement(self, name, attrs): + # pdb.set_trace() + if name == 'error': + self.stack.append(ValgrindError()) + self.state_stack.append(ValgrindParseState.ERROR) + if len(self.stack) == 0: + return + if name == 'kind': + self.state_stack.append(ValgrindParseState.KIND) + elif name == 'what': + self.state_stack.append(ValgrindParseState.WHAT) + elif name == 'auxwhat': + assert self.state() in [ValgrindParseState.ERROR, ValgrindParseState.ERROR_AUX] + self.state_stack.pop() + self.state_stack.append(ValgrindParseState.ERROR_AUX) + self.state_stack.append(ValgrindParseState.AUX_WHAT) + self.stack[-1].aux.append(ValgrindWhat()) + elif name == 'stack': + state = self.state() + assert state in [ValgrindParseState.ERROR, ValgrindParseState.ERROR_AUX] + if state == ValgrindParseState.ERROR: + self.state_stack.append(ValgrindParseState.STACK) + else: + self.state_stack.append(ValgrindParseState.STACK_AUX) + elif name == 'ip': + state = self.state() + assert state in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX] + if state == ValgrindParseState.STACK: + self.state_stack.append(ValgrindParseState.STACK_IP) + if len(self.stack[-1].what.backtrace) == 0: + self.stack[-1].what.backtrace = 'addr2line -e fdbserver.debug -p -C -f -i ' + else: + self.stack[-1].what.backtrace += ' ' + else: + self.state_stack.append(ValgrindParseState.STACK_IP_AUX) + if len(self.stack[-1].aux[-1].backtrace) == 0: + self.stack[-1].aux[-1].backtrace = 'addr2line -e fdbserver.debug -p -C -f -i ' + else: + self.stack[-1].aux[-1].backtrace += ' ' + + def endElement(self, name): + # pdb.set_trace() + if name == 'error': + self.result.append(self.stack.pop()) + self.state_stack.pop() + elif name == 'kind': + assert self.state() == ValgrindParseState.KIND + self.state_stack.pop() + elif name == 'what': + assert self.state() == ValgrindParseState.WHAT + self.state_stack.pop() + elif name == 'auxwhat': + assert self.state() == ValgrindParseState.AUX_WHAT + self.state_stack.pop() + elif name == 'stack': + assert self.state() in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX] + self.state_stack.pop() + elif name == 'ip': + self.state_stack.pop() + state = self.state() + assert state in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX] + + +def parse_valgrind_output(valgrind_out_file: Path) -> List[ValgrindError]: + handler = ValgrindHandler() + with valgrind_out_file.open('r') as f: + xml.sax.parse(f, handler) + return handler.result From cca714ef0c14f2124328ba6082a567fbb474a61b Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 23 Aug 2022 21:02:04 -0600 Subject: [PATCH 21/29] fix typo and delete simfdb dir for unseed checks --- contrib/TestHarness2/test_harness/run.py | 5 +++++ contrib/TestHarness2/test_harness/valgrind.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index f8e5aae3aa..b31122a2c5 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -328,6 +328,9 @@ class TestRun: test_plan.attributes['DeterminismCheck'] = '1' if self.test_determinism else '0' out.append(test_plan) + def delete_simdir(self): + shutil.rmtree(self.temp_path / Path('simfdb')) + def run(self): command: List[str] = [] valgrind_file: Path | None = None @@ -442,6 +445,8 @@ class TestRunner: if unseed_check and run.summary.unseed is not None: if count != 0: self.restore_sim_dir(seed + count - 1) + else: + run.delete_simdir() run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, will_restart=will_restart, buggify_enabled=buggify_enabled) diff --git a/contrib/TestHarness2/test_harness/valgrind.py b/contrib/TestHarness2/test_harness/valgrind.py index 1be8af0d18..399b47c0cc 100644 --- a/contrib/TestHarness2/test_harness/valgrind.py +++ b/contrib/TestHarness2/test_harness/valgrind.py @@ -30,7 +30,7 @@ class ValgrindParseState(enum.Enum): STACK = enum.auto() STACK_AUX = enum.auto() STACK_IP = enum.auto() - STACK_IP_AUX = enum.auto + STACK_IP_AUX = enum.auto() class ValgrindHandler(xml.sax.handler.ContentHandler): From 70fced0c87228dac7bce520cf2825e03baef302d Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Tue, 23 Aug 2022 22:43:41 -0600 Subject: [PATCH 22/29] don't do unseed check for noSim tests --- contrib/TestHarness2/test_harness/config.py | 4 ++++ contrib/TestHarness2/test_harness/run.py | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index f41697c3d9..b71101634b 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -113,6 +113,10 @@ class Config: self.stats: str | None = None self.stats_args = {'type': str, 'help': 'A base64 encoded list of statistics (used to reproduce runs)', 'required': False} + self.random_seed: int | None = None + self.random_seed_args = {'type': int, + 'help': 'Force given seed given to fdbserver -- mostly useful for debugging', + 'required': False} self.kill_seconds: int = 30 * 60 self.kill_seconds_args = {'help': 'Timeout for individual test'} self.buggify_on_ratio: float = 0.8 diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index b31122a2c5..01ac5892bc 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -429,7 +429,7 @@ class TestRunner: for file in test_files: will_restart = count + 1 < len(test_files) binary = self.binary_chooser.choose_binary(file) - unseed_check = config.random.random() < config.unseed_check_ratio + unseed_check = is_no_sim(file) and config.random.random() < config.unseed_check_ratio buggify_enabled: bool = config.random.random() < config.buggify_on_ratio if unseed_check and count != 0: # for restarting tests we will need to restore the sim2 after the first run @@ -445,8 +445,6 @@ class TestRunner: if unseed_check and run.summary.unseed is not None: if count != 0: self.restore_sim_dir(seed + count - 1) - else: - run.delete_simdir() run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0, stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed, will_restart=will_restart, buggify_enabled=buggify_enabled) @@ -460,7 +458,7 @@ class TestRunner: return result def run(self) -> bool: - seed = config.random.randint(0, 2 ** 32 - 1) + seed = config.random_seed if not None else config.random.randint(0, 2 ** 32 - 1) test_files = self.test_picker.choose_test() success = self.run_tests(test_files, seed, self.test_picker) if config.clean_up: From ffaf17c24e28c17652d57ef9a35a96d703470448 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Wed, 24 Aug 2022 09:11:36 -0600 Subject: [PATCH 23/29] some bug fixes and some debugging code --- contrib/TestHarness2/test_harness/joshua.py | 2 +- contrib/TestHarness2/test_harness/run.py | 5 ++- .../TestHarness2/test_harness/summarize.py | 40 ++++++++++++++++++- contrib/TestHarness2/test_harness/timeout.py | 2 +- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 7b152d2c2c..2257d7cb2b 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -40,7 +40,7 @@ class ToSummaryTree(xml.sax.handler.ContentHandler): self.stack[-1].children.append(closed) -def _print_summary(summary: SummaryTree, commands: Set[str]) -> str: +def _print_summary(summary: SummaryTree, commands: Set[str]): cmd = [] if config.reproduce_prefix is not None: cmd.append(config.reproduce_prefix) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 01ac5892bc..a96a344ce0 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -439,6 +439,9 @@ class TestRunner: result = result and run.success test_picker.add_time(test_files[0], run.run_time, run.summary.out) decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled) + # TODO: Remove debugging code + if unseed_check and run.summary.unseed: + run.summary.out.append(run.summary.list_simfdb()) run.summary.out.dump(sys.stdout) if not result: return False @@ -458,7 +461,7 @@ class TestRunner: return result def run(self) -> bool: - seed = config.random_seed if not None else config.random.randint(0, 2 ** 32 - 1) + seed = config.random_seed if config.random_seed is not None else config.random.randint(0, 2 ** 32 - 1) test_files = self.test_picker.choose_test() success = self.run_tests(test_files, seed, self.test_picker) if config.clean_up: diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index df79cb95c3..f73420b542 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -13,7 +13,7 @@ import xml.sax.handler import xml.sax.saxutils from pathlib import Path -from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple +from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple, Iterator, Iterable from test_harness.config import config from test_harness.valgrind import parse_valgrind_output @@ -28,7 +28,7 @@ class SummaryTree: def append(self, element: SummaryTree): self.children.append(element) - def to_dict(self, add_name: bool = True) -> Dict[str, Any] | [Any]: + def to_dict(self, add_name: bool = True) -> Dict[str, Any] | List[Any]: if len(self.children) > 0 and len(self.attributes) == 0: children = [] for child in self.children: @@ -294,6 +294,22 @@ class TraceFiles: def __len__(self) -> int: return len(self.runs) + def items(self) -> Iterator[List[Path]]: + class TraceFilesIterator(Iterable[List[Path]]): + def __init__(self, trace_files: TraceFiles): + self.current = 0 + self.trace_files: TraceFiles = trace_files + + def __iter__(self): + return self + + def __next__(self) -> List[Path]: + if len(self.trace_files) <= self.current: + raise StopIteration + self.current += 1 + return self.trace_files[self.current - 1] + return TraceFilesIterator(self) + class Summary: def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None, @@ -322,6 +338,7 @@ class Summary: self.error_out = error_out self.stderr_severity: str = '40' self.will_restart: bool = will_restart + self.test_dir: Path | None = None if uid is not None: self.out.attributes['TestUID'] = str(uid) @@ -340,6 +357,7 @@ class Summary: self.done() def summarize(self, trace_dir: Path, command: str): + self.test_dir = trace_dir trace_files = TraceFiles(trace_dir) if len(trace_files) == 0: self.error = True @@ -358,6 +376,24 @@ class Summary: test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage-metadata',), self.coverage) + def list_simfdb(self) -> SummaryTree: + res = SummaryTree('SimFDB') + res.attributes['TestDir'] = str(self.test_dir) + if self.test_dir is None: + return res + simfdb = self.test_dir / Path('simfdb') + if not simfdb.exists(): + res.attributes['NoSimDir'] = "simfdb doesn't exist" + return res + elif not simfdb.is_dir(): + res.attributes['NoSimDir'] = 'simfdb is not a directory' + return res + for file in simfdb.iterdir(): + child = SummaryTree('Directory' if file.is_dir() else 'File') + child.attributes['Name'] = file.name + res.append(child) + return res + def ok(self): return not self.error diff --git a/contrib/TestHarness2/test_harness/timeout.py b/contrib/TestHarness2/test_harness/timeout.py index 372d0795e7..e3d245ea94 100644 --- a/contrib/TestHarness2/test_harness/timeout.py +++ b/contrib/TestHarness2/test_harness/timeout.py @@ -47,7 +47,7 @@ if __name__ == '__main__': for directory in dirs_with_files_matching(Path.cwd(), re.compile(r'trace.*\.(json|xml)'), recurse=True): trace_files = TraceFiles(directory) - for files in trace_files: + for files in trace_files.items(): if config.use_valgrind: for valgrind_file in valgrind_files: summary = Summary(Path('bin/fdbserver'), was_killed=True) From fcca5cec6d3d01153c5b673185264900ee8d3376 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Wed, 24 Aug 2022 11:56:28 -0600 Subject: [PATCH 24/29] remove todo -- this change is generally useful --- contrib/TestHarness2/test_harness/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index a96a344ce0..20246c35d1 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -439,7 +439,6 @@ class TestRunner: result = result and run.success test_picker.add_time(test_files[0], run.run_time, run.summary.out) decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled) - # TODO: Remove debugging code if unseed_check and run.summary.unseed: run.summary.out.append(run.summary.list_simfdb()) run.summary.out.dump(sys.stdout) From 80cbb2568027b7a07ce865590262435188e634ed Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Wed, 24 Aug 2022 15:21:31 -0600 Subject: [PATCH 25/29] fixed stupid logical bug --- contrib/TestHarness2/test_harness/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 20246c35d1..68e94bdc0d 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -429,7 +429,7 @@ class TestRunner: for file in test_files: will_restart = count + 1 < len(test_files) binary = self.binary_chooser.choose_binary(file) - unseed_check = is_no_sim(file) and config.random.random() < config.unseed_check_ratio + unseed_check = not is_no_sim(file) and config.random.random() < config.unseed_check_ratio buggify_enabled: bool = config.random.random() < config.buggify_on_ratio if unseed_check and count != 0: # for restarting tests we will need to restore the sim2 after the first run From 5107bc337e5f5344599115a1b033605487defebc Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Thu, 25 Aug 2022 08:56:18 -0600 Subject: [PATCH 26/29] Apply suggestions from code review Co-authored-by: A.J. Beamon --- contrib/TestHarness2/test_harness/config.py | 2 +- contrib/TestHarness2/test_harness/results.py | 2 +- contrib/TestHarness2/test_harness/run.py | 4 ++-- contrib/TestHarness2/test_harness/summarize.py | 2 +- contrib/TestHarness2/test_harness/timeout.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index b71101634b..f0bfdf8d50 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -155,7 +155,7 @@ class Config: self.binary = Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver') self.binary_args = {'help': 'Path to executable'} self.hit_per_runs_ratio: int = 20000 - self.hit_per_runs_ratio_args = {'help': 'How many test runs should hit each code probe at least once'} + self.hit_per_runs_ratio_args = {'help': 'Maximum test runs before each code probe hit at least once'} self.output_format: str = 'xml' self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml'], 'help': 'What format TestHarness should produce'} diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index e3292db822..486c497d35 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -57,7 +57,7 @@ class EnsembleResults: def dump(self, prefix: str): errors = 0 out = SummaryTree('EnsembleResults') - out.attributes['TotalRunTime'] = str(self.global_statistics.total_cpu_time) + out.attributes['TotalRuntime'] = str(self.global_statistics.total_cpu_time) out.attributes['TotalTestRuns'] = str(self.global_statistics.total_test_runs) out.attributes['TotalProbesHit'] = str(self.global_statistics.total_probes_hit) out.attributes['MinProbeHit'] = str(self.min_coverage_hit) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 68e94bdc0d..714159e6d9 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -123,7 +123,7 @@ class TestPicker: times.frombytes(base64.standard_b64decode(serialized)) assert len(times) == len(self.tests.items()) idx = 0 - for _, spec in self.tests.items(): + for idx, (_, spec) in enumerate(self.tests.items()): spec.total_runtime = times[idx] idx += 1 @@ -426,7 +426,7 @@ class TestRunner: def run_tests(self, test_files: List[Path], seed: int, test_picker: TestPicker) -> bool: count = 0 result: bool = True - for file in test_files: + for count, file in enumerate(test_files): will_restart = count + 1 < len(test_files) binary = self.binary_chooser.choose_binary(file) unseed_check = not is_no_sim(file) and config.random.random() < config.unseed_check_ratio diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index f73420b542..4c268c14a5 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -416,7 +416,7 @@ class Summary: if self.errors > config.max_errors: child = SummaryTree('ErrorLimitExceeded') child.attributes['Severity'] = '40' - child.attributes['ErrorCount'] = str(self.warnings) + child.attributes['ErrorCount'] = str(self.errors) self.out.append(child) self.error_list.append('ErrorLimitExceeded') if self.was_killed: diff --git a/contrib/TestHarness2/test_harness/timeout.py b/contrib/TestHarness2/test_harness/timeout.py index e3d245ea94..5d8bef75e8 100644 --- a/contrib/TestHarness2/test_harness/timeout.py +++ b/contrib/TestHarness2/test_harness/timeout.py @@ -8,7 +8,7 @@ from test_harness.summarize import Summary, TraceFiles from typing import Pattern, List -def files_matching(path: Path, pattern: Pattern, recurs: bool = True) -> List[Path]: +def files_matching(path: Path, pattern: Pattern, recurse: bool = True) -> List[Path]: res: List[Path] = [] for file in path.iterdir(): if file.is_file() and pattern.match(file.name) is not None: From e2534e96f5e3ac427e97dc27d7974afbd66b8539 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Thu, 25 Aug 2022 11:42:55 -0600 Subject: [PATCH 27/29] Address review comments --- contrib/TestHarness2/test_harness/config.py | 4 +- contrib/TestHarness2/test_harness/fdb.py | 8 ++-- contrib/TestHarness2/test_harness/joshua.py | 7 +--- contrib/TestHarness2/test_harness/run.py | 9 ++-- .../TestHarness2/test_harness/summarize.py | 41 +------------------ contrib/TestHarness2/test_harness/timeout.py | 10 ++--- 6 files changed, 17 insertions(+), 62 deletions(-) diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index f0bfdf8d50..eb183996a7 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -163,8 +163,8 @@ class Config: self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'} self.exclude_test_files: str = r'.^' self.exclude_test_files_args = {'help': 'Don\'t consider test files whose path match against the given regex'} - self.include_test_names: str = r'.*' - self.include_test_names_args = {'help': 'Only consider tests whose names match against the given regex'} + self.include_test_classes: str = r'.*' + self.include_test_classes_args = {'help': 'Only consider tests whose names match against the given regex'} self.exclude_test_names: str = r'.^' self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'} self.details: bool = False diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py index 2e340b7f76..1e6afa3906 100644 --- a/contrib/TestHarness2/test_harness/fdb.py +++ b/contrib/TestHarness2/test_harness/fdb.py @@ -11,6 +11,9 @@ from test_harness.run import StatFetcher, TestDescription from test_harness.config import config from test_harness.summarize import SummaryTree, Coverage +# Before increasing this, make sure that all Joshua clusters (at Apple and Snowflake) have been upgraded. +# This version needs to be changed if we either need newer features from FDB or the current API version is +# getting retired. fdb.api_version(630) @@ -31,15 +34,12 @@ def open_db(cluster_file: str | None): def chunkify(iterable, sz: int): - count = 0 res = [] for item in iterable: res.append(item) - count += 1 - if count >= sz: + if len(res) >= sz: yield res res = [] - count = 0 if len(res) > 0: yield res diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 2257d7cb2b..33c5881dcc 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -62,7 +62,7 @@ def _print_summary(summary: SummaryTree, commands: Set[str]): cmd += ['-b', arg] else: cmd += ['b', ''] - cmd += ['--crash', '--trace_format', 'json'] + cmd += ['--crash', '--trace_format', config.trace_format] key = ' '.join(cmd) count = 1 while key in commands: @@ -106,10 +106,7 @@ def _print_summary(summary: SummaryTree, commands: Set[str]): if child.name == 'BuggifySection': file = child.attributes['File'] line = int(child.attributes['Line']) - if file in buggifies: - buggifies[file].append(line) - else: - buggifies[file] = [line] + buggifies.setdefault(file, []).append(line) buggifies_elem = SummaryTree('Buggifies') for file, lines in buggifies.items(): lines.sort() diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 714159e6d9..c5e948eb6d 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -63,7 +63,7 @@ class TestPicker: raise RuntimeError('{} is neither a directory nor a file'.format(test_dir)) self.include_files_regex = re.compile(config.include_test_files) self.exclude_files_regex = re.compile(config.exclude_test_files) - self.include_tests_regex = re.compile(config.include_test_names) + self.include_tests_regex = re.compile(config.include_test_classes) self.exclude_tests_regex = re.compile(config.exclude_test_names) self.test_dir: Path = test_dir self.tests: OrderedDict[str, TestDescription] = collections.OrderedDict() @@ -122,10 +122,8 @@ class TestPicker: times = array.array('I') times.frombytes(base64.standard_b64decode(serialized)) assert len(times) == len(self.tests.items()) - idx = 0 for idx, (_, spec) in enumerate(self.tests.items()): spec.total_runtime = times[idx] - idx += 1 def parse_txt(self, path: Path): if self.include_files_regex.search(str(path)) is None or self.exclude_files_regex.search(str(path)) is not None: @@ -149,7 +147,8 @@ class TestPicker: try: priority = float(kv[1]) except ValueError: - pass + raise RuntimeError("Can't parse {} -- testPriority in {} should be set to a float".format(kv[1], + path)) if test_name is not None and test_class is not None and priority is not None: break if test_name is None: @@ -424,7 +423,6 @@ class TestRunner: shutil.move(src_dir, dest_dir) def run_tests(self, test_files: List[Path], seed: int, test_picker: TestPicker) -> bool: - count = 0 result: bool = True for count, file in enumerate(test_files): will_restart = count + 1 < len(test_files) @@ -456,7 +454,6 @@ class TestRunner: result = result and run2.success if not result: return False - count += 1 return result def run(self) -> bool: diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 4c268c14a5..8b604ec08e 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -121,10 +121,7 @@ class ParseHandler: self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict() def add_handler(self, attr: Tuple[str, Optional[str]], callback: ParserCallback) -> None: - if attr in self.events: - self.events[attr].append(callback) - else: - self.events[attr] = [callback] + self.events.setdefault(attr, []).append(callback) def _call(self, callback: ParserCallback, attrs: Dict[str, str]) -> str | None: try: @@ -188,35 +185,6 @@ class JsonParser(Parser): handler.handle(obj) -def format_test_error(attrs: Dict[str, str], include_details: bool) -> str: - trace_type = attrs['Type'] - res = trace_type - if trace_type == 'InternalError': - res = '{} {} {}'.format(trace_type, attrs['File'], attrs['Line']) - elif trace_type == 'TestFailure': - res = '{} {}'.format(trace_type, attrs['Reason']) - elif trace_type == 'ValgrindError': - res = '{} {}'.format(trace_type, attrs['What']) - elif trace_type == 'ExitCode': - res = '{0} 0x{1:x}'.format(trace_type, int(attrs['Code'])) - elif trace_type == 'StdErrOutput': - res = '{}: {}'.format(trace_type, attrs['Output']) - elif trace_type == 'BTreeIntegrityCheck': - res = '{}: {}'.format(trace_type, attrs['ErrorDetail']) - for k in ['Error', 'WinErrorCode', 'LinuxErrorCode']: - if k in attrs: - res += ' {}'.format(attrs[k]) - if 'Status' in attrs: - res += ' Status={}'.format(attrs['Status']) - if 'In' in attrs: - res += ' In {}'.format(attrs['In']) - if 'SQLiteError' in attrs: - res += ' SQLiteError={0}({1})'.format(attrs['SQLiteError'], attrs['SQLiteErrorCode']) - if 'Details' in attrs and include_details: - res += ': {}'.format(attrs['Details']) - return res - - class Coverage: def __init__(self, file: str, line: str | int, comment: str | None = None): self.file = file @@ -330,7 +298,6 @@ class Summary: self.severity_map: OrderedDict[tuple[str, int], int] = collections.OrderedDict() self.error: bool = False self.errors: int = 0 - self.error_list: List[str] = [] self.warnings: int = 0 self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict() self.test_count: int = 0 @@ -361,7 +328,6 @@ class Summary: trace_files = TraceFiles(trace_dir) if len(trace_files) == 0: self.error = True - self.error_list.append('No traces produced') child = SummaryTree('NoTracesFound') child.attributes['Severity'] = '40' child.attributes['Path'] = str(trace_dir.absolute()) @@ -418,7 +384,6 @@ class Summary: child.attributes['Severity'] = '40' child.attributes['ErrorCount'] = str(self.errors) self.out.append(child) - self.error_list.append('ErrorLimitExceeded') if self.was_killed: child = SummaryTree('ExternalTimeout') child.attributes['Severity'] = '40' @@ -451,7 +416,6 @@ class Summary: _, _, exc_traceback = sys.exc_info() child.attributes['Trace'] = repr(traceback.format_tb(exc_traceback)) self.out.append(child) - self.error_list.append('Failed to parse valgrind output: {}'.format(str(e))) if not self.test_end_found: child = SummaryTree('TestUnexpectedlyNotFinished') child.attributes['Severity'] = '40' @@ -513,7 +477,6 @@ class Summary: child.attributes['Severity'] = '40' child.attributes['ErrorMessage'] = str(e) self.out.append(child) - self.error_list.append('SummarizationError {}'.format(str(e))) def register_handlers(self): def remap_event_severity(attrs): @@ -568,7 +531,6 @@ class Summary: child.attributes['Severity'] = str(severity) if severity >= 40: self.error = True - self.error_list.append('UnseedMismatch') self.out.append(child) self.out.attributes['SimElapsedTime'] = attrs['SimTime'] self.out.attributes['RealElapsedTime'] = attrs['RealTime'] @@ -598,7 +560,6 @@ class Summary: for k, v in attrs.items(): child.attributes[k] = v self.out.append(child) - self.error_list.append(format_test_error(attrs, True)) self.handler.add_handler(('Severity', '40'), parse_error) diff --git a/contrib/TestHarness2/test_harness/timeout.py b/contrib/TestHarness2/test_harness/timeout.py index 5d8bef75e8..90af7096fd 100644 --- a/contrib/TestHarness2/test_harness/timeout.py +++ b/contrib/TestHarness2/test_harness/timeout.py @@ -13,8 +13,8 @@ def files_matching(path: Path, pattern: Pattern, recurse: bool = True) -> List[P for file in path.iterdir(): if file.is_file() and pattern.match(file.name) is not None: res.append(file) - elif file.is_dir() and recurs: - res += files_matching(file, pattern, recurs) + elif file.is_dir() and recurse: + res += files_matching(file, pattern, recurse) return res @@ -29,9 +29,9 @@ def dirs_with_files_matching(path: Path, pattern: Pattern, recurse: bool = True) sub_directories.append(file) if has_file: res.append(path) - if recurse: - for file in sub_directories: - res += dirs_with_files_matching(file, pattern, recurse=True) + if recurse: + for file in sub_directories: + res += dirs_with_files_matching(file, pattern, recurse=True) res.sort() return res From 1ef7859c9a5c9d73ce4721b327a587058f7a4f39 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Thu, 25 Aug 2022 16:05:15 -0600 Subject: [PATCH 28/29] deleted empty cmake file, commented in empty py-file --- contrib/TestHarness2/CMakeLists.txt | 0 contrib/TestHarness2/test_harness/__init__.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 contrib/TestHarness2/CMakeLists.txt diff --git a/contrib/TestHarness2/CMakeLists.txt b/contrib/TestHarness2/CMakeLists.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/contrib/TestHarness2/test_harness/__init__.py b/contrib/TestHarness2/test_harness/__init__.py index 8b13789179..3cb95520ec 100644 --- a/contrib/TestHarness2/test_harness/__init__.py +++ b/contrib/TestHarness2/test_harness/__init__.py @@ -1 +1,2 @@ - +# Currently this file is left intentionally empty. It's main job for now is to indicate that this directory +# should be used as a module. From 2592ca7b2c4e02395330b1dd6559078dce1830a6 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Thu, 25 Aug 2022 16:16:21 -0600 Subject: [PATCH 29/29] remove wrong subdir include --- contrib/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ad741de86d..75ca06243f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -18,4 +18,3 @@ if(NOT WIN32) add_subdirectory(TestHarness) endif() add_subdirectory(mockkms) -add_subdirectory(TestHarness2)