canvas-lms/Jenkinsfile.rspecq

297 lines
12 KiB
Groovy

#!/usr/bin/env groovy
/*
* Copyright (C) 2022 - present Instructure, Inc.
*
* This file is part of Canvas.
*
* Canvas is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3 of the License.
*
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License along
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
commitMessageFlag.setDefaultValues(commitMessageFlagDefaults() + commitMessageFlagPrivateDefaults())
@groovy.transform.Field
def rspecqNodeTotal = 50
@groovy.transform.Field
def rspecqTargetTime = 180
@groovy.transform.Field
def summaryMessage = ''
@groovy.transform.Field
def crystalballColor = 'danger'
@groovy.transform.Field
def specUnique = []
def computeTestCountNecessary() {
copyArtifacts filter: 'tmp/crystalball_spec_list.txt', projectName: "${env.UPSTREAM}", selector: upstream()
copyArtifacts filter: 'tmp/crystalball_map_version.txt', projectName: "${env.UPSTREAM}", selector: upstream()
def specs = readFile(file: 'tmp/crystalball_spec_list.txt')
def mapVersion = readFile(file: 'tmp/crystalball_map_version.txt')
specUnique = specs.tokenize(',').unique()
env.CRYSTALBALL_SPEC = specUnique.join(' ')
summaryMessage = "Map: $mapVersion\n"
if (commitMessageFlag('skip-crystalball') as Boolean) {
summaryMessage += "Skip Crystalball Detected! - Running everything!"
crystalballColor = 'danger'
env.CRYSTALBALL_SPEC = '.'
return false
}
// Crystalball predictor returned empty text file
if (specUnique.size() == 0) {
summaryMessage += "Configuration Changes Detected! - Running everything!"
crystalballColor = 'danger'
env.CRYSTALBALL_SPEC = '.'
return false
}
// Crystalball predictor returned "."
if (specUnique.size() == 1 && specUnique[0] == '.') {
summaryMessage += "New File Detected! - Complete Suite Re-run!"
crystalballColor = 'danger'
env.CRYSTALBALL_SPEC = '.'
return false
}
return true
}
def computeTestCount() {
credentials.withStarlordCredentials { ->
sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
}
sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')
sh(script: "docker-compose exec -T canvas bundle exec rspec --dry-run \
--require './spec/formatters/node_count_formatter.rb' \
--format NodeCountRecorder \
--out formatter_out.txt --pattern ${specUnique.join(',')}", label: 'Get Node Count')
def formatterResult = sh(script: 'docker-compose exec -T canvas cat formatter_out.txt', returnStdout: true).trim().split(" ")
def nodeTotal = formatterResult[0].toInteger()
def specTotal = formatterResult[1].toInteger()
summaryMessage += "${specTotal} specs across ${specUnique.size()} files using ${nodeTotal} nodes"
echo "=== Prediction Summary ===\n${summaryMessage}\n${specUnique.join('\n')}"
crystalballColor = nodeTotal > rspecqNodeTotal ? 'warning' : 'good'
rspecqNodeTotal = nodeTotal > rspecqNodeTotal ? rspecqNodeTotal : nodeTotal
def jqAddCommand = []
specUnique.each { x ->
def globMatcher = x =~ /^(.+)\*spec\.rb$/
if (globMatcher.matches()) {
jqAddCommand.add("([with_entries(select(.key | match(\"${globMatcher.group(1)}\"))) | to_entries[] | .value] | add)")
} else {
jqAddCommand.add(".[\"./${x}\"]")
}
}
def specTimings = sh(script: """#!/usr/bin/env bash
set -ex
docker run \
-e TEST_QUEUE_HOST \
-v \$(pwd)/build/new-jenkins/iterscores.lua:/iterscores.lua \
-t \
--rm \
$REGISTRY_BASE/redis:alpine /bin/sh -c 'redis-cli -h $TEST_QUEUE_HOST -p 6379 --eval /iterscores.lua' | jq 'fromjson | ${jqAddCommand.join(' + ')}'
""", returnStdout: true)
if (specTimings && specTimings.isNumber()) {
echo "=== Total Spec Time $specTimings"
def newNodeTotal = Math.ceil(specTimings.toFloat() / (rspecqTargetTime * RSPEC_PROCESSES.toInteger()))
summaryMessage += " (new: ${newNodeTotal})"
} else {
echo "=== No Spec Timings Found! ==="
}
}
def sendCrystalballSlack(summary, color, status) {
def jobInfo = "<https://gerrit.instructure.com/$env.GERRIT_CHANGE_NUMBER|Gerrit> | <$env.BUILD_URL|Jenkins>: **$status!**"
def message = "$jobInfo\n$summary"
slackSend channel: '#crystalball-noisy', color: color, message: message
}
def sendCrystalballMetrics() {
def queueInfo = sh(script: "docker run -e TEST_QUEUE_HOST -t --rm $REGISTRY_BASE/redis:alpine /bin/sh -c '\
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:example_count;\
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:build_time'", returnStdout: true).split('\n')
def exampleCount = queueInfo[0].replaceAll('"', '').trim()
def buildTime = queueInfo[1].replaceAll('"', '').trim()
reportToSplunk('rspecq_crystalball_data', [
'node_count': rspecqNodeTotal,
'example_count': exampleCount.toInteger(),
'execution_time': buildTime.toInteger(),
'result': currentBuild.currentResult,
'upstream_tag': "${env.UPSTREAM_TAG}"])
}
def redisUrl() {
return "redis://${TEST_QUEUE_HOST}:6379"
}
pipeline {
agent none
options {
ansiColor('xterm')
timeout(60)
timestamps()
}
environment {
REGISTRY_BASE = 'starlord.inscloudgate.net/jenkins'
COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
COMPOSE_PROJECT_NAME = 'test-queue'
FORCE_FAILURE = commitMessageFlag("force-failure-rspec").asBooleanInteger()
RERUNS_RETRY = commitMessageFlag('rspecq-max-requeues').asType(Integer)
RSPEC_PROCESSES = commitMessageFlag('rspecq-processes').asType(Integer)
RSPECQ_FILE_SPLIT_THRESHOLD = commitMessageFlag('rspecq-file-split-threshold').asType(Integer)
RSPECQ_MAX_REQUEUES = commitMessageFlag('rspecq-max-requeues').asType(Integer)
TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
EXCLUDE_TESTS = '.*/(selenium/performance|instfs/selenium|contracts)'
RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
POSTGRES_PASSWORD = 'sekret'
RSPECQ_REDIS_URL = redisUrl()
}
stages {
stage('Environment') {
steps {
script {
def rspecNodeRequirements = [label: nodeLabel()]
env.CRYSTALBALL_SPEC = '.'
def postRunnerHandler = [
onStageEnded: { stageName, stageConfig, result ->
node('master') {
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
sendCrystalballSlack(summaryMessage, crystalballColor, stageConfig.status())
}
buildSummaryReport.saveRunManifest()
}
}
]
def postBuildHandler = [
onNodeReleasing: {
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
sendCrystalballMetrics()
}
}
]
def postStageHandler = [
onStageEnded: { stageName, stageConfig, result ->
buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
}
]
extendedStage('Runner').hooks(postRunnerHandler).obeysAllowStages(false).execute {
extendedStage('Builder').hooks(postStageHandler + postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
extendedStage('RSpecQ Setup').hooks(postStageHandler).obeysAllowStages(false).execute {
sh 'rm -vrf ./tmp'
checkout scm
distribution.stashBuildScripts()
}
def testCountNecessary = !configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1' && computeTestCountNecessary()
if (testCountNecessary) {
extendedStage('RSpecQ Compute Build Distribution').hooks(postStageHandler).obeysAllowStages(false).execute {
computeTestCount()
}
}
extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
def rspecqStages = [:]
def initialNodeHooks = testCountNecessary ?
postStageHandler + [onNodeReleasing: { rspecStage.tearDownNode() }] :
postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }]
extendedStage('RSpecQ Initial Node')
.envVars(['CI_NODE_INDEX=0',
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
.hooks(initialNodeHooks)
.timeout(env.NODE_TIMEOUT.toInteger())
.queue(rspecqStages) {
def initialNodeStages = [:]
extendedStage('RSpecQ Reporter').queue(initialNodeStages) {
try {
sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
--build=${JOB_NAME}_build${BUILD_NUMBER} \
--queue-wait-timeout 120 \
--redis-url $RSPECQ_REDIS_URL \
--report", label: 'Reporter')
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
/* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
sh '''#!/bin/bash
ids=($(docker ps -aq --filter "name=canvas_"))
for i in "${ids[@]}"
do
docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
done
'''
}
throw e
}
}
extendedStage("RSpecQ Set 00").queue(initialNodeStages) {
rspecStage.runRspecqSuite()
}
parallel(initialNodeStages)
}
for (int i = 1; i < rspecqNodeTotal; i++) {
def index = i
extendedStage("RSpecQ Set ${(index).toString().padLeft(2, '0')}")
.envVars(["CI_NODE_INDEX=$index",
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }])
.nodeRequirements(rspecNodeRequirements)
.timeout(env.NODE_TIMEOUT.toInteger())
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
}
parallel(rspecqStages)
} //rspecQ
} //builder
} //runner
} //script
} //steps
} //environment
} //stages
} //pipeline