From bc1a61ff84152ffadc38998b60ac24e831bfc845 Mon Sep 17 00:00:00 2001 From: Aaron Ogata Date: Wed, 3 Mar 2021 11:35:07 -0800 Subject: [PATCH] improve failure report refs DE-519 Test Plan: 1. Build successful 2. Build failure w/ no detected causes 3. Build failure w/ rspec failures 4. Build failure w/ intermittent failures 5. Buidl failure w/ stage failures [canvas-builds-refspec=fa9df2ba5173b83b29c035460fb41d3d2eba8d65] Change-Id: I55812ea5a82255d2d922817e7e641079d8d556a2 Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/259854 Tested-by: Service Cloud Jenkins QA-Review: Aaron Ogata QA-Review: Kyle Rosenbaum Product-Review: Aaron Ogata Reviewed-by: Andrea Cirulli Reviewed-by: James Butters --- Jenkinsfile | 44 ++++++++++---------- Jenkinsfile.parallel_logs | 6 +-- build/new-jenkins/groovy/distribution.groovy | 2 +- build/new-jenkins/groovy/rspec.groovy | 19 ++++++--- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 089102cea6c..d97cdaa1ade 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,6 +20,8 @@ import org.jenkinsci.plugins.workflow.support.steps.build.DownstreamFailureCause import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException +def BLUE_OCEAN_TESTS_TAB = "display/redirect?page=tests" + def buildParameters = [ string(name: 'GERRIT_REFSPEC', value: "${env.GERRIT_REFSPEC}"), string(name: 'GERRIT_EVENT_TYPE', value: "${env.GERRIT_EVENT_TYPE}"), @@ -114,9 +116,7 @@ def postFn(status) { 'requestTime': requestEndTime - requestStartTime, ]) - failureReport.publishReportFromArtifacts('Rspec Test Failures', 'rspec') - failureReport.publishReportFromArtifacts('Selenium Test Failures', 'selenium') - failureReport.submit() + buildSummaryReport.publishReport('Build Summary Report', status) if(status == 'SUCCESS' && configuration.isChangeMerged() && isPatchsetPublishable()) { dockerUtils.tagRemote(env.PATCHSET_TAG, env.MERGE_TAG) @@ -392,7 +392,7 @@ pipeline { // wait for the current steps to complete (even wait to spin up a node), causing // extremely long wait times for a restart. Investigation in DE-166 / DE-158. protectedNode('canvas-docker-nospot', { status -> cleanupFn(status) }, { status -> postFn(status) }) { - timedStage('Setup') { + buildSummaryReport.timedStageAndReportIfFailure('Setup') { timeout(time: 2) { echo "Cleaning Workspace From Previous Runs" sh 'ls -A1 | xargs rm -rf' @@ -452,7 +452,7 @@ pipeline { } if(!configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms' && !configuration.skipRebase()) { - timedStage('Rebase') { + buildSummaryReport.timedStageAndReportIfFailure('Rebase') { timeout(time: 2) { rebaseHelper(GERRIT_BRANCH) if ( GERRIT_BRANCH ==~ /dev\/.*/ ) { @@ -466,7 +466,7 @@ pipeline { } } - timedStage('Build Docker Image') { + buildSummaryReport.timedStageAndReportIfFailure('Build Docker Image') { timeout(time: 20) { if (!configuration.isChangeMerged() && configuration.skipDockerBuild()) { sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $MERGE_TAG' @@ -533,7 +533,7 @@ pipeline { } } - timedStage('Run Migrations') { + buildSummaryReport.timedStageAndReportIfFailure('Run Migrations') { timeout(time: 10) { def cacheLoadScope = configuration.isChangeMerged() || configuration.getBoolean('skip-cache') ? '' : env.IMAGE_CACHE_MERGE_SCOPE def cacheSaveScope = configuration.isChangeMerged() ? env.IMAGE_CACHE_MERGE_SCOPE : '' @@ -610,7 +610,7 @@ pipeline { if (!configuration.isChangeMerged()) { echo 'adding Linters' - timedStage('Linters', stages, { + buildSummaryReport.timedStageAndReportIfFailure('Linters', stages, { credentials.withGerritCredentials { withEnv([ "FORCE_FAILURE=${configuration.getBoolean('force-failure-linters', 'false')}", @@ -629,13 +629,13 @@ pipeline { } echo 'adding Consumer Smoke Test' - timedStage('Consumer Smoke Test', stages, { + buildSummaryReport.timedStageAndReportIfFailure('Consumer Smoke Test', stages, { sh 'build/new-jenkins/consumer-smoke-test.sh' }) echo 'adding Vendored Gems' timedStage('Vendored Gems', stages, { - failureReport.buildAndReportIfFailure('/Canvas/test-suites/vendored-gems', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/vendored-gems', buildParameters + [ string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"), string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"), string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"), @@ -644,7 +644,7 @@ pipeline { def jsReady = null - timedStage('Javascript (Build Image)', stages, { + buildSummaryReport.timedStageAndReportIfFailure('Javascript (Build Image)', stages, { try { def cacheScope = configuration.isChangeMerged() ? env.IMAGE_CACHE_MERGE_SCOPE : env.IMAGE_CACHE_BUILD_SCOPE @@ -694,10 +694,10 @@ pipeline { error "image dependency failed to build" } - failureReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ string(name: 'KARMA_RUNNER_IMAGE', value: env.KARMA_RUNNER_IMAGE), string(name: 'TEST_SUITE', value: "jest"), - ], true, "testReport") + ], true, BLUE_OCEAN_TESTS_TAB, "Javascript (Jest)") }) echo 'adding Javascript (Coffeescript)' @@ -708,10 +708,10 @@ pipeline { error "image dependency failed to build" } - failureReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ string(name: 'KARMA_RUNNER_IMAGE', value: env.KARMA_RUNNER_IMAGE), string(name: 'TEST_SUITE', value: "coffee"), - ], true, "testReport") + ], true, BLUE_OCEAN_TESTS_TAB, "Javascript (Coffeescript)") }) echo 'adding Javascript (Karma)' @@ -722,15 +722,15 @@ pipeline { error "image dependency failed to build" } - failureReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/JS', buildParameters + [ string(name: 'KARMA_RUNNER_IMAGE', value: env.KARMA_RUNNER_IMAGE), string(name: 'TEST_SUITE', value: "karma"), - ], true, "testReport") + ], true, BLUE_OCEAN_TESTS_TAB, "Javascript (Karma)") }) echo 'adding Contract Tests' timedStage('Contract Tests', stages, { - failureReport.buildAndReportIfFailure('/Canvas/test-suites/contract-tests', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/contract-tests', buildParameters + [ string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"), string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"), string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"), @@ -740,7 +740,7 @@ pipeline { if (sh(script: 'build/new-jenkins/check-for-migrations.sh', returnStatus: true) == 0) { echo 'adding CDC Schema check' timedStage('CDC Schema Check', stages, { - failureReport.buildAndReportIfFailure('/Canvas/cdc-event-transformer-master', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/cdc-event-transformer-master', buildParameters + [ string(name: 'CANVAS_LMS_IMAGE_PATH', value: "${env.PATCHSET_TAG}"), ]) }) @@ -758,7 +758,7 @@ pipeline { ) { echo 'adding Flakey Spec Catcher' timedStage('Flakey Spec Catcher', stages, { - failureReport.buildAndReportIfFailure('/Canvas/test-suites/flakey-spec-catcher', buildParameters + [ + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/flakey-spec-catcher', buildParameters + [ string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"), string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"), string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"), @@ -775,12 +775,12 @@ pipeline { if(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(dockerDevFiles, 'HEAD^')) { echo 'adding Local Docker Dev Build' timedStage('Local Docker Dev Build', stages, { - failureReport.buildAndReportIfFailure('/Canvas/test-suites/local-docker-dev-smoke', buildParameters) + buildSummaryReport.buildAndReportIfFailure('/Canvas/test-suites/local-docker-dev-smoke', buildParameters) }) } if(configuration.isChangeMerged()) { - timedStage('Dependency Check', stages, { + buildSummaryReport.timedStageAndReportIfFailure('Dependency Check', stages, { catchError (buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { try { snyk("canvas-lms:ruby", "Gemfile.lock", "$PATCHSET_TAG") diff --git a/Jenkinsfile.parallel_logs b/Jenkinsfile.parallel_logs index da7a404b874..92c2fadb15d 100644 --- a/Jenkinsfile.parallel_logs +++ b/Jenkinsfile.parallel_logs @@ -121,9 +121,9 @@ pipeline { always { script { ignoreBuildNeverStartedError { - failureReport.publishReportFromArtifacts('Rspec Test Failures', "tmp/spec_failures/rspec/**/*") - failureReport.publishReportFromArtifacts('Selenium Test Failures', "tmp/spec_failures/selenium/**/*") - failureReport.submit() + node('master') { + buildSummaryReport.publishReport('Build Summary Report', currentBuild.getResult() == 'SUCCESS' ? 'SUCCESS' : 'FAILURE') + } } } } diff --git a/build/new-jenkins/groovy/distribution.groovy b/build/new-jenkins/groovy/distribution.groovy index 5ba086fcfc6..7a45bf777d3 100644 --- a/build/new-jenkins/groovy/distribution.groovy +++ b/build/new-jenkins/groovy/distribution.groovy @@ -40,7 +40,7 @@ def appendStagesAsBuildNodes(nodes, // we cant use String.format, so... yea def stage_name = "$stage_name_prefix ${(index + 1).toString().padLeft(2, '0')}" def timeStart = new Date() - timedStage(stage_name, nodes, { + buildSummaryReport.timedStageAndReportIfFailure(stage_name, nodes, { protectedNode("canvas-docker") { echo "Running on node ${env.NODE_NAME}" def duration = TimeCategory.minus(new Date(), timeStart).toMilliseconds() diff --git a/build/new-jenkins/groovy/rspec.groovy b/build/new-jenkins/groovy/rspec.groovy index bc976ad20ba..6c982961ca2 100644 --- a/build/new-jenkins/groovy/rspec.groovy +++ b/build/new-jenkins/groovy/rspec.groovy @@ -109,10 +109,6 @@ def _runRspecTestSuite( sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers') sh(script: 'build/new-jenkins/docker-compose-rspec-parallel.sh', label: 'Run Tests') } - } catch(Exception e) { - failureReport.addFailure(prefix, "${BUILD_URL}${prefix}-test-failures") - - throw e } finally { // copy spec failures to local sh "build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures/$prefix canvas_ --allow-error --clean-dir" @@ -127,7 +123,20 @@ def _runRspecTestSuite( findFiles(glob: "tmp/spec_failures/$prefix/**/index.html").each { file -> // node_18/spec_failures/canvas__9224fba6fc34/spec_failures/Initial/spec/selenium/force_failure_spec.rb:20/index // split on the 5th to give us the rerun category (Initial, Rerun_1, Rerun_2...) - failureReport.addFailurePathByCategory(prefix, file.getPath(), file.getPath().split("/")[5]) + + def pathCategory = file.getPath().split("/")[5] + def finalCategory = reruns_retry.toInteger() == 0 ? 'Initial' : "Rerun_${reruns_retry.toInteger()}" + def splitPath = file.getPath().split('/').toList() + def specTitle = splitPath.subList(6, splitPath.size() - 1).join('/') + def artifactsPath = "../artifact/${file.getPath()}" + + buildSummaryReport.addFailurePath(specTitle, artifactsPath, pathCategory) + + if(pathCategory == finalCategory) { + buildSummaryReport.setFailureCategory(specTitle, buildSummaryReport.FAILURE_TYPE_TEST_NEVER_PASSED) + } else { + buildSummaryReport.setFailureCategoryUnlessExists(specTitle, buildSummaryReport.FAILURE_TYPE_TEST_PASSED_ON_RETRY) + } } // junit publishing will set build status to unstable if failed tests found, if so set it back to the original value