Also print test errors in results

We now attempt to import joshua.joshua_model. If this succeeds
test_harness.results will also print test errors

contrib/TestHarness2/test_harness/config.py

@@ -146,6 +146,11 @@ class Config:
         self._read_env()
         self.random.seed(self.joshua_seed, version=2)
 
+    def change_default(self, attr: str, default_val):
+        assert attr in self._config_map, 'Unknown config attribute {}'.format(attr)
+        self.__setattr__(attr, default_val)
+        self._config_map[attr].kwargs['default'] = default_val
+
     def _get_env_name(self, var_name: str) -> str:
         return self._env_names.get(var_name, 'TH_{}'.format(var_name.upper()))
 

contrib/TestHarness2/test_harness/joshua.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import sys
+import xml.sax
+import xml.sax.handler
+from typing import List
+
+from joshua import joshua_model
+from test_harness.config import config
+from test_harness.summarize import SummaryTree
+
+
+class ToSummaryTree(xml.sax.handler.ContentHandler):
+    def __init__(self):
+        super().__init__()
+        self.root: SummaryTree | None = None
+        self.stack: List[SummaryTree] = []
+
+    def result(self) -> SummaryTree:
+        assert len(self.stack) == 0 and self.root is not None, 'Parse Error'
+        return self.root
+
+    def startElement(self, name, attrs):
+        new_child = SummaryTree(name)
+        for k, v in attrs.items():
+            new_child.attributes[k] = v
+        self.stack.append(new_child)
+
+    def endElement(self, name):
+        closed = self.stack.pop()
+        assert closed.name == name
+        if len(self.stack) == 0:
+            self.root = closed
+        else:
+            self.stack[-1].children.append(closed)
+
+
+def print_errors(ensemble_id: str):
+    joshua_model.open(config.cluster_file)
+    properties = joshua_model.get_ensemble_properties(ensemble_id)
+    compressed = properties["compressed"] if "compressed" in properties else False
+    for rec in joshua_model.tail_results(ensemble_id, errors_only=(not config.details), compressed=compressed):
+        if len(rec) == 5:
+            version_stamp, result_code, host, seed, output = rec
+        elif len(rec) == 4:
+            version_stamp, result_code, host, output = rec
+            seed = None
+        elif len(rec) == 3:
+            version_stamp, result_code, output = rec
+            host = None
+            seed = None
+        elif len(rec) == 2:
+            version_stamp, seed = rec
+            output = str(joshua_model.fdb.tuple.unpack(seed)[0]) + "\n"
+            result_code = None
+            host = None
+            seed = None
+        else:
+            raise Exception("Unknown result format")
+        summary = ToSummaryTree()
+        xml.sax.parseString(output, summary)
+        res = summary.result()
+        res.dump(sys.stdout, prefix=('  ' if config.pretty_print else ''), new_line=config.pretty_print)

contrib/TestHarness2/test_harness/results.py

@@ -51,7 +51,7 @@ class EnsembleResults:
         if self.coverage_ok:
             self.coverage_ok = self.min_coverage_hit > self.ratio
 
-    def dump(self):
+    def dump(self, prefix: str):
         errors = 0
         out = SummaryTree('EnsembleResults')
         out.attributes['TotalRunTime'] = str(self.global_statistics.total_cpu_time)
@@ -83,17 +83,50 @@ class EnsembleResults:
                 out.append(child)
         if errors > 0:
             out.attributes['Errors'] = str(errors)
-        out.dump(sys.stdout)
+        out.dump(sys.stdout, prefix=prefix, new_line=config.pretty_print)
+
+
+def write_header(ensemble_id: str):
+    if config.output_format == 'json':
+        if config.pretty_print:
+            print('{')
+            print('  {}: {}'.format('ID', ensemble_id))
+        else:
+            sys.stdout.write('{{{}: {}'.format('ID', ensemble_id))
+    elif config.output_format == 'xml':
+        sys.stdout.write('<Ensemble ID="{}">'.format(ensemble_id))
+        if config.pretty_print:
+            sys.stdout.write('\n')
+    else:
+        assert False, 'unknown output format {}'.format(config.output_format)
+
+
+def write_footer():
+    if config.output_format == 'json':
+        sys.stdout.write('}')
+    elif config.output_format == 'xml':
+        sys.stdout.write('</Ensemble>')
+    else:
+        assert False, 'unknown output format {}'.format(config.output_format)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    config.change_default('pretty_print', True)
     config.build_arguments(parser)
     parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for')
     args = parser.parse_args()
     config.extract_args(args)
-    config.pretty_print = True
     config.output_format = args.output_format
+    write_header(args.ensemble_id)
+    try:
+        import test_harness.joshua
+        test_harness.joshua.print_errors(args.ensemble_id)
+    except ModuleNotFoundError:
+        child = SummaryTree('JoshuaNotFound')
+        child.attributes['Severity'] = '30'
+        child.attributes['Message'] = 'Could not import Joshua -- set PYTHONPATH to joshua checkout dir'
+        child.dump(sys.stdout, prefix=('  ' if config.pretty_print else ''), new_line=config.pretty_print)
     results = EnsembleResults(config.cluster_file, args.ensemble_id)
-    results.dump()
+    results.dump('  ' if config.pretty_print else '')
     exit(0 if results.coverage_ok else 1)

contrib/TestHarness2/test_harness/run.py

@@ -383,6 +383,19 @@ class TestRun:
         return self.summary.ok()
 
 
+def decorate_summary(out: SummaryTree, test_file: Path, seed: int, buggify: bool):
+    """Sometimes a test can crash before ProgramStart is written to the traces. These
+    tests are then hard to reproduce (they can be reproduced through TestHarness but
+    require the user to run in the joshua docker container). To account for this we
+    will write the necessary information into the attributes if it is missing."""
+    if 'TestFile' not in out.attributes:
+        out.attributes['TestFile'] = str(test_file)
+    if 'RandomSeed' not in out.attributes:
+        out.attributes['RandomSeed'] = str(seed)
+    if 'BuggifyEnabled' not in out.attributes:
+        out.attributes['BuggifyEnabled'] = '1' if buggify else '0'
+
+
 class TestRunner:
     def __init__(self):
         self.uid = uuid.uuid4()
@@ -422,6 +435,7 @@ class TestRunner:
                           stats=test_picker.dump_stats(), will_restart=will_restart)
             result = result and run.success
             test_picker.add_time(test_files[0], run.run_time, run.summary.out)
+            decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled)
             run.summary.out.dump(sys.stdout)
             if not result:
                 return False
@@ -432,8 +446,9 @@ class TestRunner:
                                stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed,
                                will_restart=will_restart)
                 test_picker.add_time(file, run2.run_time, run.summary.out)
-                run.summary.out.dump(sys.stdout)
-                result = result and run.success
+                decorate_summary(run2.summary.out, file, seed + count, run.buggify_enabled)
+                run2.summary.out.dump(sys.stdout)
+                result = result and run2.success
             count += 1
         return result
 

contrib/TestHarness2/test_harness/summarize.py

@@ -28,18 +28,22 @@ class SummaryTree:
         self.children.append(element)
 
     def to_dict(self) -> Dict[str, Any]:
-        res: Dict[str, Any] = {'Type': self.name}
-        for k, v in self.attributes.items():
-            res[k] = v
         children = []
         for child in self.children:
             children.append(child.to_dict())
+        if len(children) > 0 and len(self.attributes) == 0:
+            return {self.name: children}
+        res: Dict[str, Any] = {'Type': self.name}
+        for k, v in self.attributes.items():
+            res[k] = v
         if len(children) > 0:
             res['children'] = children
         return res
 
-    def to_json(self, out: TextIO):
-        json.dump(self.to_dict(), out, indent=('  ' if config.pretty_print else None))
+    def to_json(self, out: TextIO, prefix: str = ''):
+        res = json.dumps(self.to_dict(), indent=('  ' if config.pretty_print else None))
+        for line in res.splitlines(False):
+            out.write('{}{}\n'.format(prefix, line))
 
     def to_xml(self, out: TextIO, prefix: str = ''):
         # minidom doesn't support omitting the xml declaration which is a problem for joshua
@@ -79,14 +83,15 @@ class SummaryTree:
                 out.write('\n')
             child.to_xml(out, prefix=('  {}'.format(prefix) if config.pretty_print else prefix))
         if len(self.children) > 0:
-            out.write('{}</{}>'.format(('\n' if config.pretty_print else ''), self.name))
+            out.write('{}{}</{}>'.format(('\n' if config.pretty_print else ''), prefix, self.name))
 
-    def dump(self, out: TextIO):
+    def dump(self, out: TextIO, prefix: str = '', new_line: bool = True):
         if config.output_format == 'json':
-            self.to_json(out)
+            self.to_json(out, prefix=prefix)
         else:
-            self.to_xml(out)
-        out.write('\n')
+            self.to_xml(out, prefix=prefix)
+        if new_line:
+            out.write('\n')
 
 
 ParserCallback = Callable[[Dict[str, str]], Optional[str]]