272 lines
11 KiB
Groovy
272 lines
11 KiB
Groovy
#!/usr/bin/env groovy
|
|
|
|
/*
|
|
* Copyright (C) 2022 - present Instructure, Inc.
|
|
*
|
|
* This file is part of Canvas.
|
|
*
|
|
* Canvas is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU Affero General Public License as published by the Free
|
|
* Software Foundation, version 3 of the License.
|
|
*
|
|
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
|
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
|
* details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License along
|
|
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
|
|
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
|
|
|
|
@groovy.transform.Field
|
|
def rspecqNodeTotal = 50
|
|
@groovy.transform.Field
|
|
def summaryMessage = ''
|
|
@groovy.transform.Field
|
|
def crystalballColor = 'danger'
|
|
|
|
def setupNode() {
|
|
sh 'rm -vrf ./tmp'
|
|
|
|
checkout scm
|
|
|
|
distribution.stashBuildScripts()
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
copyArtifacts filter: 'tmp/crystalball_spec_list.txt', projectName: "${env.UPSTREAM}", selector: upstream()
|
|
copyArtifacts filter: 'tmp/crystalball_map_version.txt', projectName: "${env.UPSTREAM}", selector: upstream()
|
|
}
|
|
|
|
credentials.withStarlordCredentials { ->
|
|
sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
|
|
}
|
|
|
|
sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')
|
|
}
|
|
|
|
def exampleCount(testList) {
|
|
sh(script: "docker-compose exec -T canvas bundle exec rspec --dry-run \
|
|
--require './spec/formatters/example_count_formatter.rb' \
|
|
--format ExampleCountRecorder \
|
|
--out spec_count.txt --pattern ${testList.join(',')}", label: 'Get Test Count')
|
|
return sh(script: 'docker-compose exec -T canvas cat spec_count.txt', returnStdout: true).trim().toInteger()
|
|
}
|
|
|
|
def calcNodes(seleniumCount, seleniumPartition, rspecCount, rspecPartition) {
|
|
nodeTotal = (rspecCount + rspecPartition - 1).intdiv(rspecPartition)
|
|
nodeTotal += (seleniumCount + seleniumPartition - 1).intdiv(seleniumPartition)
|
|
rspecqNodeTotal = nodeTotal > rspecqNodeTotal ? rspecqNodeTotal : nodeTotal
|
|
}
|
|
|
|
def getColor(testCount, partition) {
|
|
if (testCount <= partition) {
|
|
return 'good'
|
|
} else if (testCount <= (partition * 4)) {
|
|
return 'warning'
|
|
}
|
|
return 'danger'
|
|
}
|
|
|
|
def computeTestCount() {
|
|
def seleniumPartition = (env.RSPEC_PROCESSES.toInteger() * 25) // 25 tests per process
|
|
def rspecPartition = (env.RSPEC_PROCESSES.toInteger() * 190) // 190 tests per process
|
|
def specs = readFile(file: 'tmp/crystalball_spec_list.txt')
|
|
def mapVersion = readFile(file: 'tmp/crystalball_map_version.txt')
|
|
def specUnique = specs.tokenize(',').unique()
|
|
def totalPrediction = specUnique.join('\n')
|
|
env.CRYSTALBALL_SPEC = specUnique.join(' ')
|
|
|
|
summaryMessage = "Map: $mapVersion\n"
|
|
|
|
if (configuration.getBoolean('skip-crystalball', 'false')) {
|
|
summaryMessage += "Skip Crystalball Detected! - Running everything!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return
|
|
}
|
|
|
|
// Crystalball predictor returned empty text file
|
|
if (specUnique.size() == 0) {
|
|
summaryMessage += "Configuration Changes Detected! - Running everything!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return
|
|
}
|
|
// Crystalball predictor returned "."
|
|
if (specUnique.size() == 1 && specUnique[0] == '.') {
|
|
summaryMessage += "New File Detected! - Complete Suite Re-run!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return
|
|
}
|
|
|
|
// split crystalball_spec
|
|
def seleniumTests = specUnique.findAll { it.contains('/selenium') }
|
|
def rspecTests = specUnique - seleniumTests
|
|
def seleniumCount = seleniumTests.size() > 0 ? exampleCount(seleniumTests) : 0
|
|
def rspecCount = rspecTests.size() > 0 ? exampleCount(rspecTests) : 0
|
|
|
|
// Format crystalball message
|
|
summaryMessage += seleniumCount > 0 ? "$seleniumCount Individual Selenium spec(s) across ${seleniumTests.size()} file(s)\n" : ''
|
|
summaryMessage += rspecCount > 0 ? "$rspecCount Individual Rspec spec(s) across ${rspecTests.size()} file(s)\n" : ''
|
|
summaryMessage += seleniumCount > 0 ? "${seleniumTests.join('\n')}\n" : ''
|
|
summaryMessage += rspecCount > 0 ? "${rspecTests.join('\n')}" : ''
|
|
|
|
crystalballColor = getColor(seleniumCount+rspecCount, seleniumPartition+rspecPartition)
|
|
|
|
echo "total prediction: \n====\n$totalPrediction\n===="
|
|
|
|
// Calculate nodes
|
|
calcNodes(seleniumCount, seleniumPartition, rspecCount, rspecPartition)
|
|
}
|
|
|
|
def sendCrystalballSlack(summary, color, status) {
|
|
def jobInfo = "<https://gerrit.instructure.com/$env.GERRIT_CHANGE_NUMBER|Gerrit> | <$env.BUILD_URL|Jenkins>: **$status!**"
|
|
def message = "$jobInfo\n$summary"
|
|
slackSend channel: '#crystalball-noisy', color: color, message: message
|
|
}
|
|
|
|
def sendCrystalballMetrics() {
|
|
def queueInfo = sh(script: "docker run -e TEST_QUEUE_HOST -t --rm $REGISTRY_BASE/redis:alpine /bin/sh -c '\
|
|
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:example_count;\
|
|
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:build_time'", returnStdout: true).split('\n')
|
|
def exampleCount = queueInfo[0].replaceAll('"', '').trim()
|
|
def buildTime = queueInfo[1].replaceAll('"', '').trim()
|
|
|
|
reportToSplunk('rspecq_crystalball_data', [
|
|
'node_count': rspecqNodeTotal,
|
|
'example_count': exampleCount.toInteger(),
|
|
'execution_time': buildTime.toInteger(),
|
|
'result': currentBuild.currentResult,
|
|
'upstream_tag': "${env.UPSTREAM_TAG}"])
|
|
}
|
|
|
|
def redisUrl() {
|
|
return "redis://${TEST_QUEUE_HOST}:6379"
|
|
}
|
|
|
|
pipeline {
|
|
agent none
|
|
options {
|
|
ansiColor('xterm')
|
|
timeout(60)
|
|
timestamps()
|
|
}
|
|
|
|
environment {
|
|
REGISTRY_BASE = 'starlord.inscloudgate.net/jenkins'
|
|
COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
|
|
FORCE_FAILURE = "${configuration.isForceFailureSelenium() ? '1' : ''}"
|
|
RERUNS_RETRY = "${configuration.getInteger('rspecq-max-requeues')}"
|
|
RSPEC_PROCESSES = "${configuration.getInteger('rspecq-processes')}"
|
|
RSPECQ_FILE_SPLIT_THRESHOLD = "${configuration.fileSplitThreshold()}"
|
|
RSPECQ_MAX_REQUEUES = "${configuration.getInteger('rspecq-max-requeues')}"
|
|
TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
|
|
EXCLUDE_TESTS = '.*/(selenium/performance|instfs/selenium|contracts)'
|
|
RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
|
|
ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
|
|
POSTGRES_PASSWORD = 'sekret'
|
|
RSPECQ_REDIS_URL = redisUrl()
|
|
}
|
|
|
|
stages {
|
|
stage('Environment') {
|
|
steps {
|
|
script {
|
|
def rspecNodeRequirements = [label: 'canvas-docker']
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
|
|
def postRunnerHandler = [
|
|
onStageEnded: { stageName, stageConfig, result ->
|
|
node('master') {
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
sendCrystalballSlack(summaryMessage, crystalballColor, stageConfig.status())
|
|
}
|
|
buildSummaryReport.saveRunManifest()
|
|
}
|
|
}
|
|
]
|
|
|
|
def postBuildHandler = [
|
|
onNodeReleasing: {
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
sendCrystalballMetrics()
|
|
}
|
|
}
|
|
]
|
|
|
|
def postStageHandler = [
|
|
onStageEnded: { stageName, stageConfig, result ->
|
|
buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
|
|
}
|
|
]
|
|
|
|
extendedStage('Runner').hooks(postRunnerHandler).obeysAllowStages(false).execute {
|
|
extendedStage('Builder').hooks(postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
|
|
stage('Setup') {
|
|
setupNode()
|
|
}
|
|
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
stage('Compute Build Distribution') {
|
|
computeTestCount()
|
|
}
|
|
}
|
|
|
|
extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
|
|
def rspecqStages = [:]
|
|
|
|
extendedStage('RSpecQ Reporter').timeout(30).queue(rspecqStages) {
|
|
try {
|
|
sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
|
|
--build=${JOB_NAME}_build${BUILD_NUMBER} \
|
|
--queue-wait-timeout 120 \
|
|
--redis-url $RSPECQ_REDIS_URL \
|
|
--report", label: 'Reporter')
|
|
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
|
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
|
|
/* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
|
|
sh '''#!/bin/bash
|
|
ids=($(docker ps -aq --filter "name=canvas_"))
|
|
for i in "${ids[@]}"
|
|
do
|
|
docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
|
|
done
|
|
'''
|
|
}
|
|
throw e
|
|
}
|
|
}
|
|
|
|
extendedStage('RSpecQ Set 00')
|
|
.envVars(['CI_NODE_INDEX=0',
|
|
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
|
|
.hooks(postStageHandler + [onNodeReleasing: { rspecStage.tearDownNode('rspec') }])
|
|
.timeout(30)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
|
|
for (int i = 1; i < rspecqNodeTotal; i++) {
|
|
def index = i
|
|
extendedStage("RSpecQ Set ${(index).toString().padLeft(2, '0')}")
|
|
.envVars(["CI_NODE_INDEX=$index",
|
|
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
|
|
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('rspec') }])
|
|
.nodeRequirements(rspecNodeRequirements)
|
|
.timeout(30)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
}
|
|
|
|
parallel(rspecqStages)
|
|
} //rspecQ
|
|
} //builder
|
|
} //runner
|
|
} //script
|
|
} //steps
|
|
} //environment
|
|
} //stages
|
|
} //pipeline
|