maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Support Unsupported Tests in xunit output 

We were reporting  "Unsupported" tests in xunit as passes, however since
they are not run, it make more sense to mark them as skipped. The Junit
xml standard has support for that, so lets use it.

llvm-svn: 332065
This commit is contained in:
Chris Matthews 2018-05-11 00:25:43 +00:00
parent 5f74591847
commit 7d6224d2fa
4 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
llvm/utils/lit/lit/Test.py
View File

@ -376,8 +376,10 @@ class Test:
testcase_xml = testcase_template.format(class_name=class_name, test_name=test_name, time=elapsed_time)
fil.write(testcase_xml)
if self.result.code.isFailure:
fil.write(">\n\t<failure >\n")
fil.write(u">\n\t<failure >\n")
fil.write(escape(self.result.output))
fil.write("\n\t</failure>\n</testcase>")
fil.write(u"\n\t</failure>\n</testcase>")
elif self.result.code == UNSUPPORTED:
fil.write(u">\n\t<skipped />\n</testcase>\n")
else:
fil.write("/>")
fil.write(u"/>")

9
llvm/utils/lit/lit/main.py
View File

@ -597,10 +597,13 @@ def main_with_tmp(builtinParameters):
by_suite[suite] = {
'passes' : 0,
'failures' : 0,
'skipped': 0,
'tests' : [] }
by_suite[suite]['tests'].append(result_test)
if result_test.result.code.isFailure:
by_suite[suite]['failures'] += 1
elif result_test.result.code == lit.Test.UNSUPPORTED:
by_suite[suite]['skipped'] += 1
else:
by_suite[suite]['passes'] += 1
xunit_output_file = open(opts.xunit_output_file, "w")
@ -610,9 +613,11 @@ def main_with_tmp(builtinParameters):
safe_suite_name = suite_name.replace(".", "-")
xunit_output_file.write("<testsuite name='" + safe_suite_name + "'")
xunit_output_file.write(" tests='" + str(suite['passes'] +
suite['failures']) + "'")
xunit_output_file.write(" failures='" + str(suite['failures']) +
suite['failures'] + suite['skipped']) + "'")
xunit_output_file.write(" failures='" + str(suite['failures']) + "'")
xunit_output_file.write(" skipped='" + str(suite['skipped']) +
"'>\n")
for result_test in suite['tests']:
result_test.writeJUnitXML(xunit_output_file)
xunit_output_file.write("\n")

17
llvm/utils/lit/tests/shtest-xunit-output.py
View File

@ -3,7 +3,7 @@
# CHECK: <?xml version="1.0" encoding="UTF-8" ?>
# CHECK-NEXT: <testsuites>
# CHECK-NEXT: <testsuite name='shtest-format' tests='23' failures='7'>
# CHECK-NEXT: <testsuite name='shtest-format' tests='23' failures='7' skipped='5'>
# CHECK: <testcase classname='shtest-format.shtest-format' name='argv0.txt' time='{{[0-9]+\.[0-9]+}}'/>
@ -32,11 +32,13 @@
# CHECK: <testcase classname='shtest-format.shtest-format' name='pass.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-any-missing.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-any-missing.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT:<skipped />
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-any-present.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-missing.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-missing.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT:<skipped />
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-present.txt' time='{{[0-9]+\.[0-9]+}}'/>
@ -46,18 +48,21 @@
# CHECK-NEXT: </testcase>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-triple.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='requires-triple.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT:<skipped />
# CHECK: <testcase classname='shtest-format.shtest-format' name='unsupported-expr-false.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='unsupported-expr-true.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.shtest-format' name='unsupported-expr-true.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT:<skipped />
# CHECK: <testcase classname='shtest-format.shtest-format' name='unsupported-star.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT: <failure{{[ ]*}}>
# CHECK: </failure>
# CHECK-NEXT: </testcase>
# CHECK: <testcase classname='shtest-format.unsupported_dir' name='some-test.txt' time='{{[0-9]+\.[0-9]+}}'/>
# CHECK: <testcase classname='shtest-format.unsupported_dir' name='some-test.txt' time='{{[0-9]+\.[0-9]+}}'>
# CHECK-NEXT:<skipped />
# CHECK: <testcase classname='shtest-format.shtest-format' name='xfail-expr-false.txt' time='{{[0-9]+\.[0-9]+}}'/>

2
llvm/utils/lit/tests/xunit-output.py
View File

@ -4,7 +4,7 @@
# CHECK: <?xml version="1.0" encoding="UTF-8" ?>
# CHECK: <testsuites>
# CHECK: <testsuite name='test-data' tests='1' failures='1'>
# CHECK: <testsuite name='test-data' tests='1' failures='1' skipped='0'>
# CHECK: <testcase classname='test-data.test-data' name='bad&amp;name.ini' time='{{[0-1]}}.{{[0-9]+}}'>
# CHECK-NEXT: <failure >
# CHECK-NEXT:&amp; &lt; &gt; "