ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move failure reports 

refs DE-348

[canvas-builds-refspec=ea37440eb5fb011f138b2e3a8239fbd70b56a8ac]

Test Plan:
1. Ensure failure report is written when tests fail
2. Ensure failuie report links work

Change-Id: Idd84536fb203fb1d525f7e566c3b55b60a110e2d
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/250992
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>
Reviewed-by: James Butters <jbutters@instructure.com>
This commit is contained in:
Aaron Ogata 2020-10-23 07:45:58 -07:00
parent 7603fd1879
commit b3e7d11c77
4 changed files with 21 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Jenkinsfile vendored
View File

@ -107,18 +107,17 @@ def isPatchsetRetriggered() {
def cleanupFn(status) {
ignoreBuildNeverStartedError {
try {
def rspec = load 'build/new-jenkins/groovy/rspec.groovy'
rspec.uploadSeleniumFailures()
rspec.uploadRSpecFailures()
failureReport.submit()
} finally {
execute 'bash/docker-cleanup.sh --allow-failure'
}
execute 'bash/docker-cleanup.sh --allow-failure'
}
}
def postFn(status) {
node('master') {
failureReport.publishReportFromArtifacts('Rspec Test Failures', "tmp/spec_failures/rspec/**/*")
failureReport.publishReportFromArtifacts('Selenium Test Failures', "tmp/spec_failures/selenium/**/*")
failureReport.submit()
}
if(status == 'FAILURE') {
maybeSlackSendFailure()
maybeRetrigger()

5
Jenkinsfile.parallel_logs
View File

@ -121,9 +121,8 @@ pipeline {
always {
script {
ignoreBuildNeverStartedError {
def rspec = load 'build/new-jenkins/groovy/rspec.groovy'
rspec.uploadSeleniumFailures()
rspec.uploadRSpecFailures()
failureReport.publishReportFromArtifacts('Rspec Test Failures', "tmp/spec_failures/rspec/**/*")
failureReport.publishReportFromArtifacts('Selenium Test Failures', "tmp/spec_failures/selenium/**/*")
failureReport.submit()
}
}

80
build/new-jenkins/groovy/reports.groovy
View File

@ -61,92 +61,12 @@ def publishSpecCoverageToS3(prefix, ci_node_total, coverage_type) {
cleanupCoverage(prefix)
}
// this method is to ensure that the stashing is done in a way that
// is expected in publishSpecFailuresAsHTML
def stashSpecFailures(prefix, index) {
dir("tmp") {
stash name: "${prefix}_spec_failures_${index}", includes: 'spec_failures/**/*', allowEmpty: true
}
}
def stashParallelLogs(prefix, index) {
dir("tmp") {
stash name: "${prefix}_spec_parallel_${index}", includes: 'parallel_runtime_rspec_tests/**/*.log'
}
}
def publishSpecFailuresAsHTML(prefix, ci_node_total, report_title) {
def htmlFiles
def failureCategories
def working_dir = "${prefix}_compiled_failures"
sh "rm -vrf ./$working_dir"
sh "mkdir -p $working_dir"
dir(working_dir) {
for(int index = 0; index < ci_node_total; index++) {
dir ("node_${index}") {
try {
unstash "${prefix}_spec_failures_${index}"
} catch(err) {
println (err)
}
}
}
htmlFiles = findFiles glob: '**/index.html'
failureCategories = buildFailureCategories(htmlFiles)
buildIndexPage(failureCategories)
htmlFiles = findFiles glob: '**/index.html'
}
def report_name = "spec-failure-$prefix"
def report_url = "${BUILD_URL}${report_name}"
archiveArtifacts(artifacts: "$working_dir/**")
publishHTML target: [
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: true,
reportDir: working_dir,
reportFiles: htmlFiles.join(','),
reportName: report_name,
reportTitles: report_title
]
sh "rm -vrf ./$working_dir"
return report_url
}
def buildFailureCategories(htmlFiles) {
Map<String, List<String>> failureCategories = [:]
if (htmlFiles.size() > 0) {
htmlFiles.each { file ->
// node_18/spec_failures/canvas__9224fba6fc34/spec_failures/Initial/spec/selenium/force_failure_spec.rb:20/index
// split on the 5th to give us the rerun category (Initial, Rerun_1, Rerun_2...)
def category = file.getPath().split("/")[4]
if (!failureCategories.containsKey(category)) {
failureCategories[category] = []
}
failureCategories[category] += file
}
}
return failureCategories
}
def buildIndexPage(failureCategories) {
def indexHtml = "<body style=\"font-family:sans-serif;line-height:1.25;font-size:14px\">"
if (failureCategories.size() < 1) {
indexHtml += "\\o/ yay good job, no failures"
} else {
failureCategories.each {category, failures ->
indexHtml += "<h1>${category} Failures</h1>"
failures.each { failure ->
def spec = (failure =~ /.*spec_failures\/(.*)\/index/)[0][1]
indexHtml += "<a href=\"${failure}\">${spec}</a><br>"
}
}
}
indexHtml += "</body>"
writeFile file: "index.html", text: indexHtml
}
def copyParallelLogs(rspecTotal, seleniumTotal) {
dir('parallel_logs') {
for(int index = 0; index < rspecTotal; index++) {

33
build/new-jenkins/groovy/rspec.groovy
View File

@ -111,13 +111,16 @@ def _runRspecTestSuite(
sh 'build/new-jenkins/docker-compose-build-up.sh'
sh 'build/new-jenkins/docker-compose-rspec-parallel.sh'
}
}
finally {
} catch(Exception e) {
failureReport.addFailure(prefix, "${BUILD_URL}${prefix}-test-failures")
throw e
} finally {
// copy spec failures to local
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures canvas_ --allow-error --clean-dir'
sh "build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures/$prefix canvas_ --allow-error --clean-dir"
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results.xml tmp/rspec_results canvas_ --allow-error --clean-dir'
def reports = load 'build/new-jenkins/groovy/reports.groovy'
reports.stashSpecFailures(prefix, index)
archiveArtifacts allowEmptyArchive: true, artifacts: "tmp/spec_failures/$prefix/**/*"
// junit publishing will set build status to unstable if failed tests found, if so set it back to the original value
def preStatus = currentBuild.rawBuild.@result
@ -128,14 +131,18 @@ def _runRspecTestSuite(
currentBuild.rawBuild.@result = preStatus
}
def reports = load 'build/new-jenkins/groovy/reports.groovy'
if (env.COVERAGE == '1') {
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/coverage/ tmp/spec_coverage canvas_ --clean-dir'
reports.stashSpecCoverage(prefix, index)
}
if (env.RSPEC_LOG == '1') {
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/parallel_runtime_rspec_tests.log ./tmp/parallel_runtime_rspec_tests canvas_ --allow-error --clean-dir'
reports.stashParallelLogs(prefix, index)
}
sh 'rm -rf ./tmp'
execute 'bash/docker-cleanup.sh --allow-failure'
}
@ -157,22 +164,6 @@ def _uploadCoverageIfSuccessful(prefix, total, coverage_name) {
}
}
def uploadSeleniumFailures() {
_uploadSpecFailures('selenium', seleniumConfig().node_total, 'Selenium Test Failures')
}
def uploadRSpecFailures() {
_uploadSpecFailures('rspec', rspecConfig().node_total, 'Rspec Test Failures')
}
def _uploadSpecFailures(prefix, total, test_name) {
def reports = load('build/new-jenkins/groovy/reports.groovy')
def report_url = reports.publishSpecFailuresAsHTML(prefix, total, test_name)
if (!successes.hasSuccessOrBuildIsSuccessful(prefix, total)) {
failureReport.addFailure(prefix, report_url)
}
}
def uploadParallelLog() {
def reports = load('build/new-jenkins/groovy/reports.groovy')
reports.copyParallelLogs(rspecConfig().node_total, seleniumConfig().node_total)