297 lines
12 KiB
Groovy
297 lines
12 KiB
Groovy
#!/usr/bin/env groovy
|
|
|
|
/*
|
|
* Copyright (C) 2022 - present Instructure, Inc.
|
|
*
|
|
* This file is part of Canvas.
|
|
*
|
|
* Canvas is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU Affero General Public License as published by the Free
|
|
* Software Foundation, version 3 of the License.
|
|
*
|
|
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
|
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
|
* details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License along
|
|
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
|
|
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
|
|
|
|
commitMessageFlag.setDefaultValues(commitMessageFlagDefaults() + commitMessageFlagPrivateDefaults())
|
|
|
|
@groovy.transform.Field
|
|
def rspecqNodeTotal = 50
|
|
@groovy.transform.Field
|
|
def rspecqTargetTime = 180
|
|
@groovy.transform.Field
|
|
def summaryMessage = ''
|
|
@groovy.transform.Field
|
|
def crystalballColor = 'danger'
|
|
@groovy.transform.Field
|
|
def specUnique = []
|
|
|
|
def computeTestCountNecessary() {
|
|
copyArtifacts filter: 'tmp/crystalball_spec_list.txt', projectName: "${env.UPSTREAM}", selector: upstream()
|
|
copyArtifacts filter: 'tmp/crystalball_map_version.txt', projectName: "${env.UPSTREAM}", selector: upstream()
|
|
|
|
def specs = readFile(file: 'tmp/crystalball_spec_list.txt')
|
|
def mapVersion = readFile(file: 'tmp/crystalball_map_version.txt')
|
|
specUnique = specs.tokenize(',').unique()
|
|
env.CRYSTALBALL_SPEC = specUnique.join(' ')
|
|
|
|
summaryMessage = "Map: $mapVersion\n"
|
|
|
|
if (commitMessageFlag('skip-crystalball') as Boolean) {
|
|
summaryMessage += "Skip Crystalball Detected! - Running everything!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return false
|
|
}
|
|
|
|
// Crystalball predictor returned empty text file
|
|
if (specUnique.size() == 0) {
|
|
summaryMessage += "Configuration Changes Detected! - Running everything!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return false
|
|
}
|
|
// Crystalball predictor returned "."
|
|
if (specUnique.size() == 1 && specUnique[0] == '.') {
|
|
summaryMessage += "New File Detected! - Complete Suite Re-run!"
|
|
crystalballColor = 'danger'
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
def computeTestCount() {
|
|
credentials.withStarlordCredentials { ->
|
|
sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
|
|
}
|
|
|
|
sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')
|
|
|
|
sh(script: "docker-compose exec -T canvas bundle exec rspec --dry-run \
|
|
--require './spec/formatters/node_count_formatter.rb' \
|
|
--format NodeCountRecorder \
|
|
--out formatter_out.txt --pattern ${specUnique.join(',')}", label: 'Get Node Count')
|
|
|
|
def formatterResult = sh(script: 'docker-compose exec -T canvas cat formatter_out.txt', returnStdout: true).trim().split(" ")
|
|
|
|
def nodeTotal = formatterResult[0].toInteger()
|
|
def specTotal = formatterResult[1].toInteger()
|
|
|
|
summaryMessage += "${specTotal} specs across ${specUnique.size()} files using ${nodeTotal} nodes"
|
|
|
|
echo "=== Prediction Summary ===\n${summaryMessage}\n${specUnique.join('\n')}"
|
|
|
|
crystalballColor = nodeTotal > rspecqNodeTotal ? 'warning' : 'good'
|
|
rspecqNodeTotal = nodeTotal > rspecqNodeTotal ? rspecqNodeTotal : nodeTotal
|
|
|
|
def jqAddCommand = []
|
|
|
|
specUnique.each { x ->
|
|
def globMatcher = x =~ /^(.+)\*spec\.rb$/
|
|
if (globMatcher.matches()) {
|
|
jqAddCommand.add("([with_entries(select(.key | match(\"${globMatcher.group(1)}\"))) | to_entries[] | .value] | add)")
|
|
} else {
|
|
jqAddCommand.add(".[\"./${x}\"]")
|
|
}
|
|
}
|
|
|
|
def specTimings = sh(script: """#!/usr/bin/env bash
|
|
set -ex
|
|
|
|
docker run \
|
|
-e TEST_QUEUE_HOST \
|
|
-v \$(pwd)/build/new-jenkins/iterscores.lua:/iterscores.lua \
|
|
-t \
|
|
--rm \
|
|
$REGISTRY_BASE/redis:alpine /bin/sh -c 'redis-cli -h $TEST_QUEUE_HOST -p 6379 --eval /iterscores.lua' | jq 'fromjson | ${jqAddCommand.join(' + ')}'
|
|
""", returnStdout: true)
|
|
|
|
if (specTimings && specTimings.isNumber()) {
|
|
echo "=== Total Spec Time $specTimings"
|
|
|
|
def newNodeTotal = Math.ceil(specTimings.toFloat() / (rspecqTargetTime * RSPEC_PROCESSES.toInteger()))
|
|
|
|
summaryMessage += " (new: ${newNodeTotal})"
|
|
} else {
|
|
echo "=== No Spec Timings Found! ==="
|
|
}
|
|
}
|
|
|
|
def sendCrystalballSlack(summary, color, status) {
|
|
def jobInfo = "<https://gerrit.instructure.com/$env.GERRIT_CHANGE_NUMBER|Gerrit> | <$env.BUILD_URL|Jenkins>: **$status!**"
|
|
def message = "$jobInfo\n$summary"
|
|
slackSend channel: '#crystalball-noisy', color: color, message: message
|
|
}
|
|
|
|
def sendCrystalballMetrics() {
|
|
def queueInfo = sh(script: "docker run -e TEST_QUEUE_HOST -t --rm $REGISTRY_BASE/redis:alpine /bin/sh -c '\
|
|
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:example_count;\
|
|
redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:build_time'", returnStdout: true).split('\n')
|
|
def exampleCount = queueInfo[0].replaceAll('"', '').trim()
|
|
def buildTime = queueInfo[1].replaceAll('"', '').trim()
|
|
|
|
reportToSplunk('rspecq_crystalball_data', [
|
|
'node_count': rspecqNodeTotal,
|
|
'example_count': exampleCount.toInteger(),
|
|
'execution_time': buildTime.toInteger(),
|
|
'result': currentBuild.currentResult,
|
|
'upstream_tag': "${env.UPSTREAM_TAG}"])
|
|
}
|
|
|
|
def redisUrl() {
|
|
return "redis://${TEST_QUEUE_HOST}:6379"
|
|
}
|
|
|
|
pipeline {
|
|
agent none
|
|
options {
|
|
ansiColor('xterm')
|
|
timeout(60)
|
|
timestamps()
|
|
}
|
|
|
|
environment {
|
|
REGISTRY_BASE = 'starlord.inscloudgate.net/jenkins'
|
|
COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
|
|
COMPOSE_PROJECT_NAME = 'test-queue'
|
|
FORCE_FAILURE = commitMessageFlag("force-failure-rspec").asBooleanInteger()
|
|
RERUNS_RETRY = commitMessageFlag('rspecq-max-requeues').asType(Integer)
|
|
RSPEC_PROCESSES = commitMessageFlag('rspecq-processes').asType(Integer)
|
|
RSPECQ_FILE_SPLIT_THRESHOLD = commitMessageFlag('rspecq-file-split-threshold').asType(Integer)
|
|
RSPECQ_MAX_REQUEUES = commitMessageFlag('rspecq-max-requeues').asType(Integer)
|
|
TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
|
|
EXCLUDE_TESTS = '.*/(selenium/performance|instfs/selenium|contracts)'
|
|
RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
|
|
ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
|
|
POSTGRES_PASSWORD = 'sekret'
|
|
RSPECQ_REDIS_URL = redisUrl()
|
|
}
|
|
|
|
stages {
|
|
stage('Environment') {
|
|
steps {
|
|
script {
|
|
def rspecNodeRequirements = [label: nodeLabel()]
|
|
env.CRYSTALBALL_SPEC = '.'
|
|
|
|
def postRunnerHandler = [
|
|
onStageEnded: { stageName, stageConfig, result ->
|
|
node('master') {
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
sendCrystalballSlack(summaryMessage, crystalballColor, stageConfig.status())
|
|
}
|
|
buildSummaryReport.saveRunManifest()
|
|
}
|
|
}
|
|
]
|
|
|
|
def postBuildHandler = [
|
|
onNodeReleasing: {
|
|
if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
|
|
sendCrystalballMetrics()
|
|
}
|
|
}
|
|
]
|
|
|
|
def postStageHandler = [
|
|
onStageEnded: { stageName, stageConfig, result ->
|
|
buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
|
|
}
|
|
]
|
|
|
|
extendedStage('Runner').hooks(postRunnerHandler).obeysAllowStages(false).execute {
|
|
extendedStage('Builder').hooks(postStageHandler + postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
|
|
extendedStage('RSpecQ Setup').hooks(postStageHandler).obeysAllowStages(false).execute {
|
|
sh 'rm -vrf ./tmp'
|
|
|
|
checkout scm
|
|
|
|
distribution.stashBuildScripts()
|
|
}
|
|
|
|
def testCountNecessary = !configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1' && computeTestCountNecessary()
|
|
|
|
if (testCountNecessary) {
|
|
extendedStage('RSpecQ Compute Build Distribution').hooks(postStageHandler).obeysAllowStages(false).execute {
|
|
|
|
computeTestCount()
|
|
}
|
|
}
|
|
|
|
extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
|
|
def rspecqStages = [:]
|
|
|
|
def initialNodeHooks = testCountNecessary ?
|
|
postStageHandler + [onNodeReleasing: { rspecStage.tearDownNode() }] :
|
|
postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }]
|
|
|
|
extendedStage('RSpecQ Initial Node')
|
|
.envVars(['CI_NODE_INDEX=0',
|
|
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
|
|
.hooks(initialNodeHooks)
|
|
.timeout(10)
|
|
.queue(rspecqStages) {
|
|
def initialNodeStages = [:]
|
|
|
|
extendedStage('RSpecQ Reporter').queue(initialNodeStages) {
|
|
try {
|
|
sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
|
|
--build=${JOB_NAME}_build${BUILD_NUMBER} \
|
|
--queue-wait-timeout 120 \
|
|
--redis-url $RSPECQ_REDIS_URL \
|
|
--report", label: 'Reporter')
|
|
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
|
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
|
|
/* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
|
|
sh '''#!/bin/bash
|
|
ids=($(docker ps -aq --filter "name=canvas_"))
|
|
for i in "${ids[@]}"
|
|
do
|
|
docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
|
|
done
|
|
'''
|
|
}
|
|
throw e
|
|
}
|
|
}
|
|
|
|
extendedStage("RSpecQ Set 00").queue(initialNodeStages) {
|
|
rspecStage.runRspecqSuite()
|
|
}
|
|
|
|
parallel(initialNodeStages)
|
|
}
|
|
|
|
for (int i = 1; i < rspecqNodeTotal; i++) {
|
|
def index = i
|
|
extendedStage("RSpecQ Set ${(index).toString().padLeft(2, '0')}")
|
|
.envVars(["CI_NODE_INDEX=$index",
|
|
"CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
|
|
"BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
|
|
.hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }])
|
|
.nodeRequirements(rspecNodeRequirements)
|
|
.timeout(10)
|
|
.queue(rspecqStages) { rspecStage.runRspecqSuite() }
|
|
}
|
|
|
|
parallel(rspecqStages)
|
|
} //rspecQ
|
|
} //builder
|
|
} //runner
|
|
} //script
|
|
} //steps
|
|
} //environment
|
|
} //stages
|
|
} //pipeline
|