basic functionality working
This commit is contained in:
Markus Pilman
parent
2704891c1a
commit
2bf6a838b8
|
|
@ -1,5 +1,4 @@
|
|||
import argparse
|
||||
import random
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
|
|
@ -10,7 +9,6 @@ from test_harness.summarize import SummaryTree
|
|||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
# seed the random number generator
|
||||
parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
config.build_arguments(parser)
|
||||
# initialize arguments
|
||||
|
|
@ -21,7 +19,6 @@ if __name__ == '__main__':
|
|||
required=False)
|
||||
args = parser.parse_args()
|
||||
config.extract_args(args)
|
||||
random.seed(config.joshua_seed)
|
||||
test_runner = TestRunner()
|
||||
if not test_runner.run(args.stats):
|
||||
exit(1)
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ class ConfigValue:
|
|||
|
||||
class Config:
|
||||
def __init__(self):
|
||||
self.random = random.Random()
|
||||
self.kill_seconds: int = 30 * 60
|
||||
self.kill_seconds_args = {'help': 'Timeout for individual test'}
|
||||
self.buggify_on_ratio: float = 0.8
|
||||
|
|
@ -102,6 +103,7 @@ class Config:
|
|||
self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'}
|
||||
self.run_dir: Path = Path('tmp')
|
||||
self.joshua_seed: int = int(os.getenv('JOSHUA_SEED', str(random.randint(0, 2 ** 32 - 1))))
|
||||
self.random.seed(self.joshua_seed, version=2)
|
||||
self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed'}
|
||||
self.print_coverage = False
|
||||
self.print_coverage_args = {'action': 'store_true'}
|
||||
|
|
@ -110,9 +112,16 @@ class Config:
|
|||
self.output_format: str = 'xml'
|
||||
self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml']}
|
||||
self.include_test_files: str = r'.*'
|
||||
self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'}
|
||||
self.exclude_test_files: str = r'.^'
|
||||
self.exclude_test_files_args = {'help': 'Don\'t consider test files whose path match against the given regex'}
|
||||
self.include_test_names: str = r'.*'
|
||||
self.include_test_names_args = {'help': 'Only consider tests whose names match against the given regex'}
|
||||
self.exclude_test_names: str = r'.^'
|
||||
self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'}
|
||||
self.max_stderr_bytes: int = 1000
|
||||
self.write_stats: bool = True
|
||||
self.read_stats: bool = True
|
||||
self.config_map = self._build_map()
|
||||
|
||||
def _build_map(self):
|
||||
|
|
@ -151,6 +160,8 @@ class Config:
|
|||
for val in self.config_map.values():
|
||||
k, v = val.get_value(args)
|
||||
config.__setattr__(k, v)
|
||||
if k == 'joshua_seed':
|
||||
self.random.seed(self.joshua_seed, version=2)
|
||||
|
||||
|
||||
config = Config()
|
||||
|
|
|
|||
|
|
@ -8,21 +8,22 @@ import os
|
|||
import resource
|
||||
import shutil
|
||||
import subprocess
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from test_harness import version
|
||||
from test_harness.config import config
|
||||
from typing import List, Pattern, Callable, OrderedDict, Dict
|
||||
from functools import total_ordering
|
||||
from pathlib import Path
|
||||
from test_harness.version import Version
|
||||
from test_harness.config import config
|
||||
from typing import List, Pattern, OrderedDict, Dict
|
||||
|
||||
from test_harness.summarize import Summary, SummaryTree
|
||||
|
||||
|
||||
@total_ordering
|
||||
class TestDescription:
|
||||
def __init__(self, path: Path, name: str, priority: float):
|
||||
self.paths: List[Path] = [path]
|
||||
|
|
@ -31,6 +32,18 @@ class TestDescription:
|
|||
# we only measure in seconds. Otherwise, keeping determinism will be difficult
|
||||
self.total_runtime: int = 0
|
||||
|
||||
def __lt__(self, other):
|
||||
if isinstance(other, TestDescription):
|
||||
return self.name < other.name
|
||||
else:
|
||||
return self.name < str(other)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, TestDescription):
|
||||
return self.name < other.name
|
||||
else:
|
||||
return self.name < str(other.name)
|
||||
|
||||
|
||||
class StatFetcher:
|
||||
def __init__(self, tests: OrderedDict[str, TestDescription]):
|
||||
|
|
@ -68,11 +81,8 @@ class FileStatsFetcher(StatFetcher):
|
|||
json.dump(self.last_state, f)
|
||||
|
||||
|
||||
StatFetcherCreator = Callable[[OrderedDict[str, TestDescription]], StatFetcher]
|
||||
|
||||
|
||||
class TestPicker:
|
||||
def __init__(self, test_dir: Path, fetcher: StatFetcherCreator):
|
||||
def __init__(self, test_dir: Path):
|
||||
if not test_dir.exists():
|
||||
raise RuntimeError('{} is neither a directory nor a file'.format(test_dir))
|
||||
self.include_files_regex = re.compile(config.include_test_files)
|
||||
|
|
@ -87,7 +97,8 @@ class TestPicker:
|
|||
for subdir in self.test_dir.iterdir():
|
||||
if subdir.is_dir() and subdir.name in config.test_dirs:
|
||||
self.walk_test_dir(subdir)
|
||||
self.fetcher = fetcher(self.tests)
|
||||
self.fetcher: StatFetcher
|
||||
# self.fetcher = fetcher(self.tests)
|
||||
|
||||
def add_time(self, test_file: Path, run_time: int) -> None:
|
||||
# getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother
|
||||
|
|
@ -99,8 +110,8 @@ class TestPicker:
|
|||
break
|
||||
if test_name is not None:
|
||||
break
|
||||
assert test_name is not None
|
||||
self.fetcher.add_run_time(test_name, run_time)
|
||||
# assert test_name is not None
|
||||
# self.fetcher.add_run_time(test_name, run_time)
|
||||
|
||||
def dump_stats(self) -> str:
|
||||
res = array.array('I')
|
||||
|
|
@ -109,7 +120,8 @@ class TestPicker:
|
|||
return base64.standard_b64encode(res.tobytes()).decode('utf-8')
|
||||
|
||||
def fetch_stats(self):
|
||||
self.fetcher.read_stats()
|
||||
# self.fetcher.read_stats()
|
||||
pass
|
||||
|
||||
def load_stats(self, serialized: str):
|
||||
times = array.array('I')
|
||||
|
|
@ -121,7 +133,7 @@ class TestPicker:
|
|||
idx += 1
|
||||
|
||||
def parse_txt(self, path: Path):
|
||||
if self.include_files_regex.match(str(path)) is None or self.exclude_files_regex.match(str(path)) is not None:
|
||||
if self.include_files_regex.search(str(path)) is None or self.exclude_files_regex.search(str(path)) is not None:
|
||||
return
|
||||
with path.open('r') as f:
|
||||
test_name: str | None = None
|
||||
|
|
@ -151,8 +163,8 @@ class TestPicker:
|
|||
test_class = test_name
|
||||
if priority is None:
|
||||
priority = 1.0
|
||||
if self.include_tests_regex.match(test_class) is None \
|
||||
or self.exclude_tests_regex.match(test_class) is not None:
|
||||
if self.include_tests_regex.search(test_class) is None \
|
||||
or self.exclude_tests_regex.search(test_class) is not None:
|
||||
return
|
||||
if test_class not in self.tests:
|
||||
self.tests[test_class] = TestDescription(path, test_class, priority)
|
||||
|
|
@ -193,9 +205,10 @@ class TestPicker:
|
|||
candidates = [v]
|
||||
elif this_time == min_runtime:
|
||||
candidates.append(v)
|
||||
choice = random.randint(0, len(candidates) - 1)
|
||||
candidates.sort()
|
||||
choice = config.random.randint(0, len(candidates) - 1)
|
||||
test = candidates[choice]
|
||||
result = test.paths[random.randint(0, len(test.paths) - 1)]
|
||||
result = test.paths[config.random.randint(0, len(test.paths) - 1)]
|
||||
if self.restart_test.match(result.name):
|
||||
return self.list_restart_files(result)
|
||||
else:
|
||||
|
|
@ -206,7 +219,7 @@ class OldBinaries:
|
|||
def __init__(self):
|
||||
self.first_file_expr = re.compile(r'.*-1\.(txt|toml)')
|
||||
self.old_binaries_path: Path = config.old_binaries_path
|
||||
self.binaries: OrderedDict[version.Version, Path] = collections.OrderedDict()
|
||||
self.binaries: OrderedDict[Version, Path] = collections.OrderedDict()
|
||||
if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir():
|
||||
return
|
||||
exec_pattern = re.compile(r'fdbserver-\d+\.\d+\.\d+(\.exe)?')
|
||||
|
|
@ -220,13 +233,14 @@ class OldBinaries:
|
|||
version_str = file.name.split('-')[1]
|
||||
if version_str.endswith('.exe'):
|
||||
version_str = version_str[0:-len('.exe')]
|
||||
self.binaries[version.Version.parse(version_str)] = file
|
||||
ver = Version.parse(version_str)
|
||||
self.binaries[ver] = file
|
||||
|
||||
def choose_binary(self, test_file: Path) -> Path:
|
||||
if len(self.binaries) == 0:
|
||||
return config.binary
|
||||
max_version = version.Version.max_version()
|
||||
min_version = version.Version.parse('5.0.0')
|
||||
max_version = Version.max_version()
|
||||
min_version = Version.parse('5.0.0')
|
||||
dirs = test_file.parent.parts
|
||||
if 'restarting' not in dirs:
|
||||
return config.binary
|
||||
|
|
@ -239,16 +253,16 @@ class OldBinaries:
|
|||
# upgrade test -- we only return an old version for the first test file
|
||||
return config.binary
|
||||
if version_expr[0] == 'from' or version_expr[0] == 'to':
|
||||
min_version = version.Version.parse(version_expr[1])
|
||||
min_version = Version.parse(version_expr[1])
|
||||
if len(version_expr) == 4 and version_expr[2] == 'until':
|
||||
max_version = version.Version.parse(version_expr[3])
|
||||
max_version = Version.parse(version_expr[3])
|
||||
candidates: List[Path] = []
|
||||
for ver, binary in self.binaries.items():
|
||||
if min_version <= ver <= max_version:
|
||||
candidates.append(binary)
|
||||
if len(candidates) == 0:
|
||||
return config.binary
|
||||
return random.choice(candidates)
|
||||
return config.random.choice(candidates)
|
||||
|
||||
|
||||
def is_restarting_test(test_file: Path):
|
||||
|
|
@ -287,7 +301,7 @@ class ResourceMonitor(threading.Thread):
|
|||
class TestRun:
|
||||
def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID,
|
||||
restarting: bool = False, test_determinism: bool = False,
|
||||
stats: str | None = None, expected_unseed: int | None = None):
|
||||
stats: str | None = None, expected_unseed: int | None = None, will_restart: bool = False):
|
||||
self.binary = binary
|
||||
self.test_file = test_file
|
||||
self.random_seed = random_seed
|
||||
|
|
@ -299,13 +313,16 @@ class TestRun:
|
|||
self.err_out: str = 'error.xml'
|
||||
self.use_valgrind: bool = config.use_valgrind
|
||||
self.old_binary_path: Path = config.old_binaries_path
|
||||
self.buggify_enabled: bool = random.random() < config.buggify_on_ratio
|
||||
self.buggify_enabled: bool = config.random.random() < config.buggify_on_ratio
|
||||
self.fault_injection_enabled: bool = True
|
||||
self.trace_format = config.trace_format
|
||||
if Version.of_binary(self.binary) < "6.1.0":
|
||||
self.trace_format = None
|
||||
self.temp_path = config.run_dir / str(self.uid)
|
||||
# state for the run
|
||||
self.retryable_error: bool = False
|
||||
self.summary: Summary = Summary(binary)
|
||||
self.summary: Summary = Summary(binary, uid=self.uid, stats=self.stats, expected_unseed=self.expected_unseed,
|
||||
will_restart=will_restart)
|
||||
self.run_time: int = 0
|
||||
self.success = self.run()
|
||||
|
||||
|
|
@ -332,9 +349,11 @@ class TestRun:
|
|||
command += [str(self.binary.absolute()),
|
||||
'-r', 'test' if is_no_sim(self.test_file) else 'simulation',
|
||||
'-f', str(self.test_file),
|
||||
'-s', str(self.random_seed),
|
||||
'-fi', 'on' if self.fault_injection_enabled else 'off',
|
||||
'--trace_format', self.trace_format]
|
||||
'-s', str(self.random_seed)]
|
||||
if self.trace_format is not None:
|
||||
command += ['--trace_format', self.trace_format]
|
||||
if Version.of_binary(self.binary) >= '7.1.0':
|
||||
command += ['-fi', 'on' if self.fault_injection_enabled else 'off']
|
||||
if self.restarting:
|
||||
command.append('--restarting')
|
||||
if self.buggify_enabled:
|
||||
|
|
@ -347,12 +366,16 @@ class TestRun:
|
|||
# self.log_test_plan(out)
|
||||
resources = ResourceMonitor()
|
||||
resources.start()
|
||||
process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.temp_path)
|
||||
process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=self.temp_path,
|
||||
text=True)
|
||||
did_kill = False
|
||||
timeout = 20 * config.kill_seconds if self.use_valgrind else config.kill_seconds
|
||||
err_out: str
|
||||
try:
|
||||
process.wait(20 * config.kill_seconds if self.use_valgrind else config.kill_seconds)
|
||||
_, err_out = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
_, err_out = process.communicate()
|
||||
did_kill = True
|
||||
resources.stop()
|
||||
resources.join()
|
||||
|
|
@ -360,10 +383,9 @@ class TestRun:
|
|||
self.summary.runtime = resources.time()
|
||||
self.summary.max_rss = resources.max_rss
|
||||
self.summary.was_killed = did_kill
|
||||
self.summary = Summary(self.binary, runtime=resources.time(), max_rss=resources.max_rss,
|
||||
was_killed=did_kill, uid=self.uid, stats=self.stats, valgrind_out_file=valgrind_file,
|
||||
expected_unseed=self.expected_unseed)
|
||||
self.summary.summarize(self.temp_path)
|
||||
self.summary.valgrind_out_file = valgrind_file
|
||||
self.summary.error_out = err_out
|
||||
self.summary.summarize(self.temp_path, ' '.join(command))
|
||||
self.summary.out.dump(sys.stdout)
|
||||
return self.summary.ok()
|
||||
|
||||
|
|
@ -371,19 +393,11 @@ class TestRun:
|
|||
class TestRunner:
|
||||
def __init__(self):
|
||||
self.uid = uuid.uuid4()
|
||||
self.test_path: str = 'tests'
|
||||
self.test_path: Path = Path('tests')
|
||||
self.cluster_file: str | None = None
|
||||
self.fdb_app_dir: str | None = None
|
||||
self.stat_fetcher: StatFetcherCreator = \
|
||||
lambda x: FileStatsFetcher(os.getenv('JOSHUA_STAT_FILE', 'stats.json'), x)
|
||||
self.binary_chooser = OldBinaries()
|
||||
|
||||
def fetch_stats_from_fdb(self, cluster_file: str, app_dir: str):
|
||||
def fdb_fetcher(tests: OrderedDict[str, TestDescription]):
|
||||
from . import fdb
|
||||
return fdb.FDBStatFetcher(cluster_file, app_dir, tests)
|
||||
|
||||
self.stat_fetcher = fdb_fetcher
|
||||
self.test_picker = TestPicker(self.test_path)
|
||||
|
||||
def backup_sim_dir(self, seed: int):
|
||||
temp_dir = config.run_dir / str(self.uid)
|
||||
|
|
@ -405,34 +419,37 @@ class TestRunner:
|
|||
count = 0
|
||||
result: bool = True
|
||||
for file in test_files:
|
||||
will_restart = count + 1 < len(test_files)
|
||||
binary = self.binary_chooser.choose_binary(file)
|
||||
unseed_check = random.random() < config.unseed_check_ratio
|
||||
unseed_check = config.random.random() < config.unseed_check_ratio
|
||||
if unseed_check and count != 0:
|
||||
# for restarting tests we will need to restore the sim2 after the first run
|
||||
self.backup_sim_dir(seed + count - 1)
|
||||
run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
|
||||
stats=test_picker.dump_stats())
|
||||
stats=test_picker.dump_stats(), will_restart=will_restart)
|
||||
result = result and run.success
|
||||
test_picker.add_time(file, run.run_time)
|
||||
if run.success and unseed_check and run.summary.unseed is not None:
|
||||
self.restore_sim_dir(seed + count - 1)
|
||||
run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
|
||||
stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed)
|
||||
test_picker.add_time(file, run2.run_time)
|
||||
test_picker.add_time(test_files[0], run.run_time)
|
||||
if not result:
|
||||
return False
|
||||
if unseed_check and run.summary.unseed is not None:
|
||||
if count != 0:
|
||||
self.restore_sim_dir(seed + count - 1)
|
||||
run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
|
||||
stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed,
|
||||
will_restart=will_restart)
|
||||
test_picker.add_time(file, run2.run_time)
|
||||
result = result and run.success
|
||||
count += 1
|
||||
return result
|
||||
|
||||
def run(self, stats: str | None) -> bool:
|
||||
seed = random.randint(0, 2 ** 32 - 1)
|
||||
test_picker = TestPicker(Path(self.test_path), self.stat_fetcher)
|
||||
seed = config.random.randint(0, 2 ** 32 - 1)
|
||||
if stats is not None:
|
||||
test_picker.load_stats(stats)
|
||||
self.test_picker.load_stats(stats)
|
||||
else:
|
||||
test_picker.fetch_stats()
|
||||
test_files = test_picker.choose_test()
|
||||
success = self.run_tests(test_files, seed, test_picker)
|
||||
self.test_picker.fetch_stats()
|
||||
test_files = self.test_picker.choose_test()
|
||||
success = self.run_tests(test_files, seed, self.test_picker)
|
||||
if config.clean_up:
|
||||
shutil.rmtree(config.run_dir / str(self.uid))
|
||||
return success
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import xml.sax.saxutils
|
|||
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple
|
||||
from xml.dom import minidom
|
||||
|
||||
from test_harness.config import config
|
||||
|
||||
|
|
@ -46,7 +45,7 @@ class SummaryTree:
|
|||
# However, our xml is very simple and therefore serializing manually is easy enough
|
||||
attrs = []
|
||||
for k, v in self.attributes.items():
|
||||
attrs.append('{}="{}"'.format(k, xml.sax.saxutils.escape(v)))
|
||||
attrs.append('{}={}'.format(k, xml.sax.saxutils.quoteattr(v)))
|
||||
elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' '))
|
||||
out.write(elem)
|
||||
if config.pretty_print:
|
||||
|
|
@ -92,7 +91,7 @@ class ParseHandler:
|
|||
self.out = out
|
||||
self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict()
|
||||
|
||||
def add_handler(self, attr: Tuple[str, str], callback: ParserCallback) -> None:
|
||||
def add_handler(self, attr: Tuple[str, Optional[str]], callback: ParserCallback) -> None:
|
||||
if attr in self.events:
|
||||
self.events[attr].append(callback)
|
||||
else:
|
||||
|
|
@ -319,7 +318,7 @@ class TraceFiles:
|
|||
else:
|
||||
self.timestamps.append(ts)
|
||||
self.runs[ts] = [file]
|
||||
self.timestamps.sort()
|
||||
self.timestamps.sort(reverse=True)
|
||||
|
||||
def __getitem__(self, idx: int) -> List[Path]:
|
||||
res = self.runs[self.timestamps[idx]]
|
||||
|
|
@ -333,7 +332,8 @@ class TraceFiles:
|
|||
class Summary:
|
||||
def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None,
|
||||
was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None,
|
||||
exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None):
|
||||
exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None,
|
||||
error_out: str = None, will_restart: bool = False):
|
||||
self.binary = binary
|
||||
self.runtime: float = runtime
|
||||
self.max_rss: int | None = max_rss
|
||||
|
|
@ -353,17 +353,21 @@ class Summary:
|
|||
self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict()
|
||||
self.test_count: int = 0
|
||||
self.tests_passed: int = 0
|
||||
self.error_out = error_out
|
||||
self.stderr_severity: str = '40'
|
||||
self.will_restart: bool = will_restart
|
||||
|
||||
if uid is not None:
|
||||
self.out.attributes['TestUID'] = str(uid)
|
||||
if stats is not None:
|
||||
self.out.attributes['Statistics'] = stats
|
||||
self.out.attributes['JoshuaSeed'] = str(config.joshua_seed)
|
||||
self.out.attributes['WillRestart'] = '1' if self.will_restart else '0'
|
||||
|
||||
self.handler = ParseHandler(self.out)
|
||||
self.register_handlers()
|
||||
|
||||
def summarize(self, trace_dir: Path):
|
||||
def summarize(self, trace_dir: Path, command: str):
|
||||
trace_files = TraceFiles(trace_dir)
|
||||
if len(trace_files) == 0:
|
||||
self.error = True
|
||||
|
|
@ -371,14 +375,15 @@ class Summary:
|
|||
child = SummaryTree('NoTracesFound')
|
||||
child.attributes['Severity'] = '40'
|
||||
child.attributes['Path'] = str(trace_dir.absolute())
|
||||
child.attributes['Command'] = command
|
||||
self.out.append(child)
|
||||
return
|
||||
for f in trace_files[0]:
|
||||
self.parse_file(f)
|
||||
self.done()
|
||||
|
||||
def ok(self):
|
||||
return not self.error and self.tests_passed == self.test_count and self.tests_passed >= 0\
|
||||
and self.test_end_found
|
||||
return not self.error
|
||||
|
||||
def done(self):
|
||||
if config.print_coverage:
|
||||
|
|
@ -429,7 +434,38 @@ class Summary:
|
|||
child = SummaryTree('TestUnexpectedlyNotFinished')
|
||||
child.attributes['Severity'] = '40'
|
||||
self.out.append(child)
|
||||
if self.error_out is not None and len(self.error_out) > 0:
|
||||
if self.stderr_severity == '40':
|
||||
self.error = True
|
||||
lines = self.error_out.split('\n')
|
||||
stderr_bytes = 0
|
||||
for line in lines:
|
||||
remaining_bytes = config.max_stderr_bytes - stderr_bytes
|
||||
if remaining_bytes > 0:
|
||||
out_err = line[0:remaining_bytes] + ('...' if len(line) > remaining_bytes else '')
|
||||
child = SummaryTree('StdErrOutput')
|
||||
child.attributes['Severity'] = self.stderr_severity
|
||||
child.attributes['Output'] = out_err
|
||||
self.out.append(child)
|
||||
stderr_bytes += len(line)
|
||||
if stderr_bytes > config.max_stderr_bytes:
|
||||
child = SummaryTree('StdErrOutputTruncated')
|
||||
child.attributes['Severity'] = self.stderr_severity
|
||||
child.attributes['BytesRemaining'] = stderr_bytes - config.max_stderr_bytes
|
||||
self.out.append(child)
|
||||
|
||||
self.out.attributes['Ok'] = '1' if self.ok() else '0'
|
||||
if not self.ok():
|
||||
reason = 'Unknown'
|
||||
if self.error:
|
||||
reason = 'ProducedErrors'
|
||||
elif not self.test_end_found:
|
||||
reason = 'TestDidNotFinish'
|
||||
elif self.tests_passed == 0:
|
||||
reason = 'NoTestsPassed'
|
||||
elif self.test_count != self.tests_passed:
|
||||
reason = 'Expected {} tests to pass, but only {} did'.format(self.test_count, self.tests_passed)
|
||||
self.out.attributes['FailReason'] = reason
|
||||
|
||||
def parse_file(self, file: Path):
|
||||
parser: Parser
|
||||
|
|
@ -581,3 +617,8 @@ class Summary:
|
|||
self.out.append(child)
|
||||
self.handler.add_handler(('Type', 'BuggifySection'), buggify_section)
|
||||
self.handler.add_handler(('Type', 'FaultInjected'), buggify_section)
|
||||
|
||||
def stderr_severity(attrs: Dict[str, str]):
|
||||
if 'NewSeverity' in attrs:
|
||||
self.stderr_severity = attrs['NewSeverity']
|
||||
self.handler.add_handler(('Type', 'StderrSeverity'), stderr_severity)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
from functools import total_ordering
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
@total_ordering
|
||||
class Version:
|
||||
def __init__(self):
|
||||
self.major: int = 0
|
||||
|
|
@ -7,24 +13,39 @@ class Version:
|
|||
def version_tuple(self):
|
||||
return self.major, self.minor, self.patch
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.version_tuple() == other.version_tuple()
|
||||
def _compare(self, other) -> int:
|
||||
lhs: Tuple[int, int, int] = self.version_tuple()
|
||||
rhs: Tuple[int, int, int]
|
||||
if isinstance(other, Version):
|
||||
rhs = other.version_tuple()
|
||||
else:
|
||||
rhs = Version.parse(str(other)).version_tuple()
|
||||
if lhs < rhs:
|
||||
return -1
|
||||
elif lhs > rhs:
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.version_tuple() < other.version_tuple()
|
||||
def __eq__(self, other) -> bool:
|
||||
return self._compare(other) == 0
|
||||
|
||||
def __le__(self, other):
|
||||
return self.version_tuple() <= other.version_tuple()
|
||||
|
||||
def __gt__(self, other):
|
||||
return self.version_tuple() > other.version_tuple()
|
||||
|
||||
def __ge__(self, other):
|
||||
return self.version_tuple() >= other.version_tuple()
|
||||
def __lt__(self, other) -> bool:
|
||||
return self._compare(other) < 0
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.version_tuple())
|
||||
|
||||
def __str__(self):
|
||||
return format('{}.{}.{}'.format(self.major, self.minor, self.patch))
|
||||
|
||||
@staticmethod
|
||||
def of_binary(binary: Path):
|
||||
parts = binary.name.split('-')
|
||||
if len(parts) != 2:
|
||||
return Version.max_version()
|
||||
return Version.parse(parts[1])
|
||||
|
||||
@staticmethod
|
||||
def parse(version: str):
|
||||
version_tuple = version.split('.')
|
||||
|
|
@ -34,6 +55,7 @@ class Version:
|
|||
self.minor = int(version_tuple[1])
|
||||
if len(version_tuple) > 2:
|
||||
self.patch = int(version_tuple[2])
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def max_version():
|
||||
|
|
@ -41,3 +63,4 @@ class Version:
|
|||
self.major = 2**32 - 1
|
||||
self.minor = 2**32 - 1
|
||||
self.patch = 2**32 - 1
|
||||
return self
|
||||
|
|
|
|||
Loading…
Reference in New Issue