283 lines
12 KiB
Groovy
283 lines
12 KiB
Groovy
#!/usr/bin/env groovy
|
|
|
|
/*
|
|
* Copyright (C) 2022 - present Instructure, Inc.
|
|
*
|
|
* This file is part of Canvas.
|
|
*
|
|
* Canvas is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU Affero General Public License as published by the Free
|
|
* Software Foundation, version 3 of the License.
|
|
*
|
|
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
|
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
|
* details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License along
|
|
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
|
|
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
|
|
|
|
@groovy.transform.Field
|
|
def rspecqNodeTotal = 23
|
|
@groovy.transform.Field
|
|
def rspecNodeTotal = 27
|
|
|
|
def setupNode() {
|
|
sh 'rm -vrf ./tmp'
|
|
|
|
checkout scm
|
|
|
|
distribution.stashBuildScripts()
|
|
|
|
credentials.withStarlordCredentials { ->
|
|
sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
|
|
}
|
|
|
|
sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')
|
|
}
|
|
|
|
def getPatchsetTag() {
|
|
(env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
|
|
}
|
|
|
|
def redisUrl() {
|
|
return "redis://${TEST_QUEUE_HOST}:6379"
|
|
}
|
|
|
|
def generateSkippedSpecsReport() {
|
|
try{
|
|
copyArtifacts(
|
|
filter: 'tmp/rspec_results/**',
|
|
optional: false,
|
|
projectName: env.JOB_NAME,
|
|
selector: specific(env.BUILD_NUMBER),
|
|
)
|
|
|
|
withEnv(['COMPOSE_FILE=docker-compose.new-jenkins.yml']) {
|
|
withCredentials([usernamePassword(credentialsId: 'INSENG_CANVAS_CI_AWS_ACCESS', usernameVariable: 'INSENG_AWS_ACCESS_KEY_ID', passwordVariable: 'INSENG_AWS_SECRET_ACCESS_KEY')]) {
|
|
def awsCreds = "AWS_DEFAULT_REGION=us-west-2 AWS_ACCESS_KEY_ID=${INSENG_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${INSENG_AWS_SECRET_ACCESS_KEY}"
|
|
sh "$awsCreds aws s3 cp s3://instructure-canvas-ci/skipped_specs_ruby.json skipped_specs.json"
|
|
sh """
|
|
docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/rspec_results/:/tmp/rspec_results \
|
|
-v \$(pwd)/\$LOCAL_WORKDIR/skipped_specs.json/:/usr/src/app/skipped_specs.json \
|
|
--name skipped-spec-collator canvas bash -c \
|
|
"bundle install; ruby build/new-jenkins/skipped_specs_manager.rb ruby"
|
|
"""
|
|
sh 'docker cp skipped-spec-collator:/tmp/skipped_specs.json skipped_specs.json'
|
|
sh "$awsCreds aws s3 cp skipped_specs.json s3://instructure-canvas-ci/skipped_specs_ruby.json"
|
|
}
|
|
sendSkippedSpecsSlackReport()
|
|
archiveArtifacts allowEmptyArchive: true, artifacts: 'skipped_specs.json'
|
|
}
|
|
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
|
slackSend channel: '#canvas-test-stats', color: 'danger', message: "<$env.BUILD_URL|coverage-ruby failed to generate skipped specs report!>"
|
|
}
|
|
}
|
|
|
|
def sendSkippedSpecsSlackReport() {
|
|
def rubySkippedSpecs = sh(script: "grep -o 'file_indicator' skipped_specs.json | wc -l", returnStdout: true).trim() ?: '0'
|
|
def color = 'danger'
|
|
if(rubySkippedSpecs.toInteger() < 100) {
|
|
color = 'good'
|
|
} else if (rubySkippedSpecs.toInteger() < 300) {
|
|
color = 'warning'
|
|
}
|
|
def jobInfo = "<$env.BUILD_URL|Ruby>"
|
|
slackSend channel: '#canvas-test-stats', color: color, message: "$rubySkippedSpecs skipped specs in $jobInfo! "
|
|
}
|
|
|
|
pipeline {
|
|
agent none
|
|
options {
|
|
ansiColor('xterm')
|
|
timeout(60)
|
|
timestamps()
|
|
}
|
|
|
|
environment {
|
|
COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
|
|
FORCE_FAILURE = "${configuration.isForceFailureSelenium() ? '1' : ''}"
|
|
RERUNS_RETRY = "${configuration.getInteger('rspecq-max-requeues')}"
|
|
RSPECQ_FILE_SPLIT_THRESHOLD = "${configuration.fileSplitThreshold()}"
|
|
RSPECQ_MAX_REQUEUES = "${configuration.getInteger('rspecq-max-requeues')}"
|
|
SELENIUM_TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/selenium'
|
|
TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
|
|
EXCLUDE_TESTS = '.*/(selenium|contracts)'
|
|
RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
|
|
ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
|
|
POSTGRES_PASSWORD = 'sekret'
|
|
RSPECQ_REDIS_URL = redisUrl()
|
|
PATCHSET_TAG = getPatchsetTag()
|
|
|
|
CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
|
|
DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
|
|
POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')
|
|
|
|
IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()
|
|
RSPEC_PROCESSES = 6
|
|
|
|
CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
|
DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
|
POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
|
|
|
COVERAGE_LOCATION = "${env.COVERAGE_TYPE == 'ruby-selenium' ? 'canvas__master__selenium--coverage/coverage' : (env.COVERAGE_TYPE == 'ruby-nonselenium' ? 'canvas__master__rspec--coverage/coverage' : 'canvas-lms-rspec/coverage')}"
|
|
}
|
|
|
|
stages {
|
|
stage('Environment') {
|
|
steps {
|
|
script {
|
|
def rspecNodeRequirements = [label: 'canvas-docker']
|
|
|
|
def postBuildHandler = [
|
|
onNodeReleasing: { stageName, stageConfig, result ->
|
|
buildSummaryReport.saveRunManifest()
|
|
|
|
copyArtifacts(
|
|
filter: 'tmp/coverage/**',
|
|
optional: false,
|
|
projectName: env.JOB_NAME,
|
|
selector: specific(env.BUILD_NUMBER),
|
|
)
|
|
|
|
withEnv(['COMPOSE_FILE=docker-compose.new-jenkins.yml']) {
|
|
sh """
|
|
docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/coverage/:/tmp/coverage \
|
|
--name coverage-collator canvas bash -c \
|
|
"bundle install; bundle exec rake coverage:report['/tmp/coverage/canvas__*/**']"
|
|
"""
|
|
|
|
sh 'docker cp coverage-collator:/usr/src/app/coverage/ coverage'
|
|
|
|
archiveArtifacts allowEmptyArchive: true, artifacts: 'coverage/**'
|
|
|
|
publishHTML target: [
|
|
allowMissing: false,
|
|
alwaysLinkToLastBuild: false,
|
|
keepAll: true,
|
|
reportDir: './coverage',
|
|
reportFiles: 'index.html',
|
|
reportName: 'Ruby Coverage Report'
|
|
]
|
|
|
|
uploadCoverage([
|
|
uploadSource: '/coverage',
|
|
uploadDest: env.COVERAGE_LOCATION
|
|
])
|
|
}
|
|
if (env.GERRIT_EVENT_TYPE != 'comment-added' && env.COVERAGE_TYPE == 'ruby-total') {
|
|
generateSkippedSpecsReport()
|
|
}
|
|
}
|
|
]
|
|
|
|
def postStageHandler = [
|
|
onStageEnded: { stageName, stageConfig, result ->
|
|
buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
|
|
}
|
|
]
|
|
|
|
extendedStage('Runner').obeysAllowStages(false).execute {
|
|
extendedStage('Builder').hooks(postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
|
|
stage('Setup') {
|
|
setupNode()
|
|
}
|
|
|
|
extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
|
|
def rspecqStages = [:]
|
|
|
|
if (env.COVERAGE_TYPE != 'ruby-selenium') {
|
|
extendedStage('RSpecQ Reporter for Rspec').timeout(30).queue(rspecqStages) {
|
|
try {
|
|
sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
|
|
--build=${JOB_NAME}_build${BUILD_NUMBER}_rspec \
|
|
--queue-wait-timeout 240 \
|
|
--redis-url $RSPECQ_REDIS_URL \
|
|
--report", label: 'Reporter')
|
|
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
|
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
|
|
/* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
|
|
sh '''#!/bin/bash
|
|
ids=( $(docker ps -aq --filter "name=canvas_") )
|
|
for i in "${ids[@]}"
|
|
do
|
|
docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
|
|
done
|
|
'''
|
|
}
|
|
throw e
|
|
}
|
|
}
|
|
}
|
|
|
|
if (env.COVERAGE_TYPE != 'ruby-nonselenium') {
|
|
extendedStage('RSpecQ Reporter for Selenium').timeout(30).queue(rspecqStages) {
|
|
try {
|
|
sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
|
|
--build=${JOB_NAME}_build${BUILD_NUMBER}_selenium \
|
|
--queue-wait-timeout 120 \
|
|
--redis-url $RSPECQ_REDIS_URL \
|
|
--report", label: 'Reporter')
|
|
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
|
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
|
|
/* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
|
|
sh '''#!/bin/bash
|
|
ids=($(docker ps -aq --filter "name=canvas_"))
|
|
for i in "${ids[@]}"
|
|
do
|
|
docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
|
|
done
|
|
'''
|
|
}
|
|
throw e
|
|
}
|
|
}
|
|
|
|
extendedStage('RSpecQ Selenium Set 00')
|
|
.envVars(['CI_NODE_INDEX=0',
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_selenium",
|
|
"TEST_PATTERN=${env.SELENIUM_TEST_PATTERN}",
|
|
'EXCLUDE_TESTS=.*/(selenium/performance|instfs/selenium|contracts)'])
|
|
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('selenium') }])
|
|
.timeout(30)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
|
|
for (int i = 1; i < rspecqNodeTotal; i++) {
|
|
def index = i
|
|
extendedStage("RSpecQ Selenium Set ${(index).toString().padLeft(2, '0')}")
|
|
.envVars(["CI_NODE_INDEX=$index",
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_selenium",
|
|
"TEST_PATTERN=${env.SELENIUM_TEST_PATTERN}",
|
|
'EXCLUDE_TESTS=.*/(selenium/performance|instfs/selenium|contracts)'])
|
|
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('selenium') }])
|
|
.nodeRequirements(rspecNodeRequirements)
|
|
.timeout(30)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
}
|
|
}
|
|
|
|
if (env.COVERAGE_TYPE != 'ruby-selenium') {
|
|
rspecNodeTotal.times { index ->
|
|
extendedStage("RSpecQ Rspec Set ${(index + 1).toString().padLeft(2, '0')}")
|
|
.envVars(["CI_NODE_INDEX=$index", "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_rspec"])
|
|
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('rspec') }])
|
|
.nodeRequirements(rspecNodeRequirements)
|
|
.timeout(30)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
}
|
|
}
|
|
|
|
parallel(rspecqStages)
|
|
} //rspecQ
|
|
} //builder
|
|
} //runner
|
|
} //script
|
|
} //steps
|
|
} //environment
|
|
} //stages
|
|
} //pipeline
|