diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 92510618b0..0e205cf1db 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -70,6 +70,7 @@ class ConfigValue: return self.name, args.__getattribute__(self.get_arg_name()) +# TODO: document this class class Config: def __init__(self): self.random = random.Random() diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py index 53d5f9d279..f4215918a7 100644 --- a/contrib/TestHarness2/test_harness/joshua.py +++ b/contrib/TestHarness2/test_harness/joshua.py @@ -1,10 +1,12 @@ from __future__ import annotations +import collections +import io import sys import xml.sax import xml.sax.handler from pathlib import Path -from typing import List +from typing import List, OrderedDict, Set from joshua import joshua_model @@ -38,7 +40,7 @@ class ToSummaryTree(xml.sax.handler.ContentHandler): self.stack[-1].children.append(closed) -def _print_summary(summary: SummaryTree): +def _print_summary(summary: SummaryTree, commands: Set[str]) -> str: cmd = [] if config.reproduce_prefix is not None: cmd.append(config.reproduce_prefix) @@ -49,6 +51,10 @@ def _print_summary(summary: SummaryTree): cmd += ['-r', role, '-f', file_name] else: cmd += ['-r', 'simulation', '-f', ''] + if 'RandomSeed' in summary.attributes: + cmd += ['-s', summary.attributes['RandomSeed']] + else: + cmd += ['-s', ''] if 'BuggifyEnabled' in summary.attributes: arg = 'on' if summary.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']: @@ -57,12 +63,75 @@ def _print_summary(summary: SummaryTree): else: cmd += ['b', ''] cmd += ['--crash', '--trace_format', 'json'] + key = ' '.join(cmd) + count = 1 + while key in commands: + key = '{} # {}'.format(' '.join(cmd), count) + count += 1 # we want the command as the first attribute attributes = {'Command': ' '.join(cmd)} for k, v in summary.attributes.items(): - attributes[k] = v + if k == 'Errors': + attributes['ErrorCount'] = v + else: + attributes[k] = v summary.attributes = attributes - summary.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) + if config.details: + key = str(len(commands)) + str_io = io.StringIO() + summary.dump(str_io, prefix=(' ' if config.pretty_print else '')) + if config.output_format == 'json': + sys.stdout.write('{}"Test{}": {}'.format(' ' if config.pretty_print else '', + key, str_io.getvalue())) + else: + sys.stdout.write(str_io.getvalue()) + if config.pretty_print: + sys.stdout.write('\n' if config.output_format == 'xml' else ',\n') + return key + error_count = 0 + warning_count = 0 + small_summary = SummaryTree('Test') + small_summary.attributes = attributes + errors = SummaryTree('Errors') + warnings = SummaryTree('Warnings') + buggifies: OrderedDict[str, List[int]] = collections.OrderedDict() + for child in summary.children: + if 'Severity' in child.attributes and child.attributes['Severity'] == '40' and error_count < config.max_errors: + error_count += 1 + errors.append(child) + if 'Severity' in child.attributes and child.attributes[ + 'Severity'] == '30' and warning_count < config.max_warnings: + warning_count += 1 + warnings.append(child) + if child.name == 'BuggifySection': + file = child.attributes['File'] + line = int(child.attributes['Line']) + if file in buggifies: + buggifies[file].append(line) + else: + buggifies[file] = [line] + buggifies_elem = SummaryTree('Buggifies') + for file, lines in buggifies.items(): + lines.sort() + if config.output_format == 'json': + buggifies_elem.attributes[file] = ' '.join(str(line) for line in lines) + else: + child = SummaryTree('Buggify') + child.attributes['File'] = file + child.attributes['Lines'] = ' '.join(str(line) for line in lines) + small_summary.append(child) + small_summary.children.append(buggifies_elem) + if len(errors.children) > 0: + small_summary.children.append(errors) + if len(warnings.children) > 0: + small_summary.children.append(warnings) + output = io.StringIO() + small_summary.dump(output, prefix=(' ' if config.pretty_print else '')) + if config.output_format == 'json': + sys.stdout.write('{}"{}": {}'.format(' ' if config.pretty_print else '', key, output.getvalue().strip())) + else: + sys.stdout.write(output.getvalue().strip()) + sys.stdout.write('\n' if config.output_format == 'xml' else ',\n') def print_errors(ensemble_id: str): @@ -88,7 +157,8 @@ def print_errors(ensemble_id: str): else: raise Exception("Unknown result format") lines = output.splitlines() + commands: Set[str] = set() for line in lines: summary = ToSummaryTree() xml.sax.parseString(line, summary) - _print_summary(summary.result()) + commands.add(_print_summary(summary.result(), commands)) diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index a2695bbe31..e3292db822 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -1,14 +1,16 @@ from __future__ import annotations +import argparse +import io +import json import re import sys -from typing import List, Tuple, OrderedDict +import test_harness.fdb +from typing import List, Tuple, OrderedDict from test_harness.summarize import SummaryTree, Coverage from test_harness.config import config - -import argparse -import test_harness.fdb +from xml.sax.saxutils import quoteattr class GlobalStatistics: @@ -84,18 +86,25 @@ class EnsembleResults: out.append(child) if errors > 0: out.attributes['Errors'] = str(errors) - out.dump(sys.stdout, prefix=prefix, new_line=config.pretty_print) + str_io = io.StringIO() + out.dump(str_io, prefix=prefix, new_line=config.pretty_print) + if config.output_format == 'xml': + sys.stdout.write(str_io.getvalue()) + else: + sys.stdout.write('{}"EnsembleResults":{}{}'.format(' ' if config.pretty_print else '', + '\n' if config.pretty_print else ' ', + str_io.getvalue())) def write_header(ensemble_id: str): if config.output_format == 'json': if config.pretty_print: print('{') - print(' {}: {}'.format('ID', ensemble_id)) + print(' "{}": {},\n'.format('ID', json.dumps(ensemble_id.strip()))) else: - sys.stdout.write('{{{}: {}'.format('ID', ensemble_id)) + sys.stdout.write('{{{}: {},'.format('ID', json.dumps(ensemble_id.strip()))) elif config.output_format == 'xml': - sys.stdout.write(''.format(ensemble_id)) + sys.stdout.write(''.format(quoteattr(ensemble_id.strip()))) if config.pretty_print: sys.stdout.write('\n') else: @@ -103,10 +112,10 @@ def write_header(ensemble_id: str): def write_footer(): - if config.output_format == 'json': - sys.stdout.write('}') - elif config.output_format == 'xml': - sys.stdout.write('') + if config.output_format == 'xml': + sys.stdout.write('\n') + elif config.output_format == 'json': + sys.stdout.write('}\n') else: assert False, 'unknown output format {}'.format(config.output_format) @@ -114,6 +123,7 @@ def write_footer(): if __name__ == '__main__': parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.change_default('pretty_print', True) + config.change_default('max_warnings', 0) config.build_arguments(parser) parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for') args = parser.parse_args() @@ -130,4 +140,5 @@ if __name__ == '__main__': child.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print) results = EnsembleResults(config.cluster_file, args.ensemble_id) results.dump(' ' if config.pretty_print else '') + write_footer() exit(0 if results.coverage_ok else 1) diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 805e566d63..efad73d88a 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -27,15 +27,32 @@ class SummaryTree: def append(self, element: SummaryTree): self.children.append(element) - def to_dict(self) -> Dict[str, Any]: - children = [] - for child in self.children: - children.append(child.to_dict()) - if len(children) > 0 and len(self.attributes) == 0: - return {self.name: children} - res: Dict[str, Any] = {'Type': self.name} + def to_dict(self, add_name: bool = True) -> Dict[str, Any] | [Any]: + if len(self.children) > 0 and len(self.attributes) == 0: + children = [] + for child in self.children: + children.append(child.to_dict()) + if add_name: + return {self.name: children} + else: + return children + res: Dict[str, Any] = {} + if add_name: + res['Type'] = self.name for k, v in self.attributes.items(): res[k] = v + children = [] + child_keys: Dict[str, int] = {} + for child in self.children: + if child.name in child_keys: + child_keys[child.name] += 1 + else: + child_keys[child.name] = 1 + for child in self.children: + if child_keys[child.name] == 1 and child.name not in self.attributes: + res[child.name] = child.to_dict(add_name=False) + else: + children.append(child.to_dict()) if len(children) > 0: res['children'] = children return res