wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test_harness.result generates valid json

This commit is contained in:
Markus Pilman 2022-08-22 20:38:42 -06:00
parent 798ae8913c
commit 41ac372931
4 changed files with 123 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/TestHarness2/test_harness/config.py
View File

@ -70,6 +70,7 @@ class ConfigValue:
return self.name, args.__getattribute__(self.get_arg_name())
# TODO: document this class
class Config:
def __init__(self):
self.random = random.Random()

80
contrib/TestHarness2/test_harness/joshua.py
View File

@ -1,10 +1,12 @@
from __future__ import annotations
import collections
import io
import sys
import xml.sax
import xml.sax.handler
from pathlib import Path
from typing import List
from typing import List, OrderedDict, Set
from joshua import joshua_model
@ -38,7 +40,7 @@ class ToSummaryTree(xml.sax.handler.ContentHandler):
self.stack[-1].children.append(closed)
def _print_summary(summary: SummaryTree):
def _print_summary(summary: SummaryTree, commands: Set[str]) -> str:
cmd = []
if config.reproduce_prefix is not None:
cmd.append(config.reproduce_prefix)
@ -49,6 +51,10 @@ def _print_summary(summary: SummaryTree):
cmd += ['-r', role, '-f', file_name]
else:
cmd += ['-r', 'simulation', '-f', '<ERROR>']
if 'RandomSeed' in summary.attributes:
cmd += ['-s', summary.attributes['RandomSeed']]
else:
cmd += ['-s', '<Error>']
if 'BuggifyEnabled' in summary.attributes:
arg = 'on'
if summary.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']:
@ -57,12 +63,75 @@ def _print_summary(summary: SummaryTree):
else:
cmd += ['b', '<ERROR>']
cmd += ['--crash', '--trace_format', 'json']
key = ' '.join(cmd)
count = 1
while key in commands:
key = '{} # {}'.format(' '.join(cmd), count)
count += 1
# we want the command as the first attribute
attributes = {'Command': ' '.join(cmd)}
for k, v in summary.attributes.items():
attributes[k] = v
if k == 'Errors':
attributes['ErrorCount'] = v
else:
attributes[k] = v
summary.attributes = attributes
summary.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print)
if config.details:
key = str(len(commands))
str_io = io.StringIO()
summary.dump(str_io, prefix=(' ' if config.pretty_print else ''))
if config.output_format == 'json':
sys.stdout.write('{}"Test{}": {}'.format(' ' if config.pretty_print else '',
key, str_io.getvalue()))
else:
sys.stdout.write(str_io.getvalue())
if config.pretty_print:
sys.stdout.write('\n' if config.output_format == 'xml' else ',\n')
return key
error_count = 0
warning_count = 0
small_summary = SummaryTree('Test')
small_summary.attributes = attributes
errors = SummaryTree('Errors')
warnings = SummaryTree('Warnings')
buggifies: OrderedDict[str, List[int]] = collections.OrderedDict()
for child in summary.children:
if 'Severity' in child.attributes and child.attributes['Severity'] == '40' and error_count < config.max_errors:
error_count += 1
errors.append(child)
if 'Severity' in child.attributes and child.attributes[
'Severity'] == '30' and warning_count < config.max_warnings:
warning_count += 1
warnings.append(child)
if child.name == 'BuggifySection':
file = child.attributes['File']
line = int(child.attributes['Line'])
if file in buggifies:
buggifies[file].append(line)
else:
buggifies[file] = [line]
buggifies_elem = SummaryTree('Buggifies')
for file, lines in buggifies.items():
lines.sort()
if config.output_format == 'json':
buggifies_elem.attributes[file] = ' '.join(str(line) for line in lines)
else:
child = SummaryTree('Buggify')
child.attributes['File'] = file
child.attributes['Lines'] = ' '.join(str(line) for line in lines)
small_summary.append(child)
small_summary.children.append(buggifies_elem)
if len(errors.children) > 0:
small_summary.children.append(errors)
if len(warnings.children) > 0:
small_summary.children.append(warnings)
output = io.StringIO()
small_summary.dump(output, prefix=(' ' if config.pretty_print else ''))
if config.output_format == 'json':
sys.stdout.write('{}"{}": {}'.format(' ' if config.pretty_print else '', key, output.getvalue().strip()))
else:
sys.stdout.write(output.getvalue().strip())
sys.stdout.write('\n' if config.output_format == 'xml' else ',\n')
def print_errors(ensemble_id: str):
@ -88,7 +157,8 @@ def print_errors(ensemble_id: str):
else:
raise Exception("Unknown result format")
lines = output.splitlines()
commands: Set[str] = set()
for line in lines:
summary = ToSummaryTree()
xml.sax.parseString(line, summary)
_print_summary(summary.result())
commands.add(_print_summary(summary.result(), commands))

35
contrib/TestHarness2/test_harness/results.py
View File

@ -1,14 +1,16 @@
from __future__ import annotations
import argparse
import io
import json
import re
import sys
from typing import List, Tuple, OrderedDict
import test_harness.fdb
from typing import List, Tuple, OrderedDict
from test_harness.summarize import SummaryTree, Coverage
from test_harness.config import config
import argparse
import test_harness.fdb
from xml.sax.saxutils import quoteattr
class GlobalStatistics:
@ -84,18 +86,25 @@ class EnsembleResults:
out.append(child)
if errors > 0:
out.attributes['Errors'] = str(errors)
out.dump(sys.stdout, prefix=prefix, new_line=config.pretty_print)
str_io = io.StringIO()
out.dump(str_io, prefix=prefix, new_line=config.pretty_print)
if config.output_format == 'xml':
sys.stdout.write(str_io.getvalue())
else:
sys.stdout.write('{}"EnsembleResults":{}{}'.format(' ' if config.pretty_print else '',
'\n' if config.pretty_print else ' ',
str_io.getvalue()))
def write_header(ensemble_id: str):
if config.output_format == 'json':
if config.pretty_print:
print('{')
print(' {}: {}'.format('ID', ensemble_id))
print(' "{}": {},\n'.format('ID', json.dumps(ensemble_id.strip())))
else:
sys.stdout.write('{{{}: {}'.format('ID', ensemble_id))
sys.stdout.write('{{{}: {},'.format('ID', json.dumps(ensemble_id.strip())))
elif config.output_format == 'xml':
sys.stdout.write('<Ensemble ID="{}">'.format(ensemble_id))
sys.stdout.write('<Ensemble ID={}>'.format(quoteattr(ensemble_id.strip())))
if config.pretty_print:
sys.stdout.write('\n')
else:
@ -103,10 +112,10 @@ def write_header(ensemble_id: str):
def write_footer():
if config.output_format == 'json':
sys.stdout.write('}')
elif config.output_format == 'xml':
sys.stdout.write('</Ensemble>')
if config.output_format == 'xml':
sys.stdout.write('</Ensemble>\n')
elif config.output_format == 'json':
sys.stdout.write('}\n')
else:
assert False, 'unknown output format {}'.format(config.output_format)
@ -114,6 +123,7 @@ def write_footer():
if __name__ == '__main__':
parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.change_default('pretty_print', True)
config.change_default('max_warnings', 0)
config.build_arguments(parser)
parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for')
args = parser.parse_args()
@ -130,4 +140,5 @@ if __name__ == '__main__':
child.dump(sys.stdout, prefix=(' ' if config.pretty_print else ''), new_line=config.pretty_print)
results = EnsembleResults(config.cluster_file, args.ensemble_id)
results.dump(' ' if config.pretty_print else '')
write_footer()
exit(0 if results.coverage_ok else 1)

31
contrib/TestHarness2/test_harness/summarize.py
View File

@ -27,15 +27,32 @@ class SummaryTree:
def append(self, element: SummaryTree):
self.children.append(element)
def to_dict(self) -> Dict[str, Any]:
children = []
for child in self.children:
children.append(child.to_dict())
if len(children) > 0 and len(self.attributes) == 0:
return {self.name: children}
res: Dict[str, Any] = {'Type': self.name}
def to_dict(self, add_name: bool = True) -> Dict[str, Any] | [Any]:
if len(self.children) > 0 and len(self.attributes) == 0:
children = []
for child in self.children:
children.append(child.to_dict())
if add_name:
return {self.name: children}
else:
return children
res: Dict[str, Any] = {}
if add_name:
res['Type'] = self.name
for k, v in self.attributes.items():
res[k] = v
children = []
child_keys: Dict[str, int] = {}
for child in self.children:
if child.name in child_keys:
child_keys[child.name] += 1
else:
child_keys[child.name] = 1
for child in self.children:
if child_keys[child.name] == 1 and child.name not in self.attributes:
res[child.name] = child.to_dict(add_name=False)
else:
children.append(child.to_dict())
if len(children) > 0:
res['children'] = children
return res