From 1b51cbddf42c2a7a069bf36710fcefe2c1fd6f7b Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Thu, 5 Oct 2017 17:32:06 +0000 Subject: [PATCH] [Analyzer Tests] Run static analyzer integration tests until the end, Do not stop at the first failure. Differential Revision: https://reviews.llvm.org/D38589 llvm-svn: 314992 --- clang/utils/analyzer/SATestBuild.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/clang/utils/analyzer/SATestBuild.py b/clang/utils/analyzer/SATestBuild.py index 26c519543e8b..f2d2df6a67b3 100755 --- a/clang/utils/analyzer/SATestBuild.py +++ b/clang/utils/analyzer/SATestBuild.py @@ -570,7 +570,11 @@ def runCmpResults(Dir, Strictness=0): 0 - success if there are no crashes or analyzer failure. 1 - success if there are no difference in the number of reported bugs. 2 - success if all the bug reports are identical. + + :return success: Whether tests pass according to the Strictness + criteria. """ + TestsPassed = True TBegin = time.time() RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) @@ -586,8 +590,6 @@ def runCmpResults(Dir, Strictness=0): RefList.remove(RefLogDir) NewList.remove(os.path.join(NewDir, LogFolderName)) - if len(RefList) == 0 or len(NewList) == 0: - return False assert(len(RefList) == len(NewList)) # There might be more then one folder underneath - one per each scan-build @@ -624,15 +626,15 @@ def runCmpResults(Dir, Strictness=0): (NumDiffs, DiffsPath,) if Strictness >= 2 and NumDiffs > 0: print "Error: Diffs found in strict mode (2)." - sys.exit(-1) + TestsPassed = False elif Strictness >= 1 and ReportsInRef != ReportsInNew: print "Error: The number of results are different in "\ "strict mode (1)." - sys.exit(-1) + TestsPassed = False print "Diagnostic comparison complete (time: %.2f)." % ( time.time() - TBegin) - return (NumDiffs > 0) + return TestsPassed def cleanupReferenceResults(SBOutputDir): @@ -686,6 +688,11 @@ def updateSVN(Mode, PMapFile): def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): + """ + Test a given project. + :return TestsPassed: Whether tests have passed according + to the :param Strictness: criteria. + """ print " \n\n--- Building project %s" % (ID,) TBegin = time.time() @@ -704,11 +711,13 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): if IsReferenceBuild: cleanupReferenceResults(SBOutputDir) + TestsPassed = True else: - runCmpResults(Dir, Strictness) + TestsPassed = runCmpResults(Dir, Strictness) print "Completed tests for project %s (time: %.2f)." % \ (ID, (time.time() - TBegin)) + return TestsPassed def isCommentCSVLine(Entries): @@ -749,6 +758,7 @@ def validateProjectFile(PMapFile): def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0): + TestsPassed = True with projectFileHandler() as PMapFile: validateProjectFile(PMapFile) @@ -760,7 +770,7 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0): # Test the projects. for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile): - testProject( + TestsPassed &= testProject( ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness) # Re-add reference results to SVN. @@ -793,4 +803,6 @@ if __name__ == '__main__': IsReference = True UpdateSVN = True - testAll(IsReference, UpdateSVN, Strictness) + TestsPassed = testAll(IsReference, UpdateSVN, Strictness) + if not TestsPassed: + sys.exit(-1)