2019-02-16 04:16:49 +08:00
#!/usr/bin/env groovy
2019-03-13 23:54:18 +08:00
/*
* Copyright (C) 2019 - present Instructure, Inc.
*
* This file is part of Canvas.
*
* Canvas is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3 of the License.
*
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License along
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
2020-04-01 01:25:11 +08:00
import org.jenkinsci.plugins.workflow.support.steps.build.DownstreamFailureCause
import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
2020-02-15 06:02:23 +08:00
def buildParameters = [
2019-08-15 02:43:09 +08:00
string(name: 'GERRIT_REFSPEC', value: "${env.GERRIT_REFSPEC}"),
string(name: 'GERRIT_EVENT_TYPE', value: "${env.GERRIT_EVENT_TYPE}"),
2020-01-18 04:26:48 +08:00
string(name: 'GERRIT_PROJECT', value: "${env.GERRIT_PROJECT}"),
2019-08-15 02:43:09 +08:00
string(name: 'GERRIT_BRANCH', value: "${env.GERRIT_BRANCH}"),
string(name: 'GERRIT_CHANGE_NUMBER', value: "${env.GERRIT_CHANGE_NUMBER}"),
string(name: 'GERRIT_PATCHSET_NUMBER', value: "${env.GERRIT_PATCHSET_NUMBER}"),
string(name: 'GERRIT_EVENT_ACCOUNT_NAME', value: "${env.GERRIT_EVENT_ACCOUNT_NAME}"),
2019-11-16 02:12:18 +08:00
string(name: 'GERRIT_EVENT_ACCOUNT_EMAIL', value: "${env.GERRIT_EVENT_ACCOUNT_EMAIL}"),
string(name: 'GERRIT_CHANGE_COMMIT_MESSAGE', value: "${env.GERRIT_CHANGE_COMMIT_MESSAGE}"),
string(name: 'GERRIT_HOST', value: "${env.GERRIT_HOST}"),
2019-12-20 02:55:08 +08:00
string(name: 'GERGICH_PUBLISH', value: "${env.GERGICH_PUBLISH}"),
string(name: 'MASTER_BOUNCER_RUN', value: "${env.MASTER_BOUNCER_RUN}")
2019-08-15 02:43:09 +08:00
]
2020-08-28 20:08:22 +08:00
def getDockerWorkDir() {
return env.GERRIT_PROJECT == "canvas-lms" ? "/usr/src/app" : "/usr/src/app/gems/plugins/${env.GERRIT_PROJECT}"
}
def getLocalWorkDir() {
return env.GERRIT_PROJECT == "canvas-lms" ? "." : "gems/plugins/${env.GERRIT_PROJECT}"
}
2019-11-23 04:57:11 +08:00
def skipIfPreviouslySuccessful(name, block) {
2020-08-14 01:21:44 +08:00
if (env.CANVAS_LMS_REFSPEC && !env.CANVAS_LMS_REFSPEC.contains('master')) {
name+="${env.CANVAS_LMS_REFSPEC}"
}
2020-10-03 05:49:51 +08:00
if (env.SKIP_CACHE.toBoolean()) {
echo "Build cache is disabled; forcing step ${name} to run, even if it was already successful."
block()
} else {
successes.skipIfPreviouslySuccessful(name, block)
}
2020-02-15 06:02:23 +08:00
}
2020-04-30 00:44:23 +08:00
def wrapBuildExecution(jobName, parameters, propagate, urlExtra) {
2020-04-01 01:25:11 +08:00
try {
2020-04-30 00:44:23 +08:00
build(job: jobName, parameters: parameters, propagate: propagate)
2020-04-01 01:25:11 +08:00
}
catch(FlowInterruptedException ex) {
// if its this type, then that means its a build failure.
// other reasons can be user cancelling or jenkins aborting, etc...
def failure = ex.causes.find { it instanceof DownstreamFailureCause }
if (failure != null) {
def downstream = failure.getDownstreamBuild()
def url = downstream.getAbsoluteUrl() + urlExtra
2020-11-02 23:14:04 +08:00
failureReport.addFailure(jobName, url)
2020-04-01 01:25:11 +08:00
}
throw ex
}
}
2020-04-23 01:30:45 +08:00
// if the build never starts or gets into a node block, then we
// can never load a file. and a very noisy/confusing error is thrown.
def ignoreBuildNeverStartedError(block) {
try {
block()
}
catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) {
if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) {
throw ex
}
else {
echo "ignored MissingContextVariableException: \n${ex.message}"
}
// we can ignore this very noisy error
}
}
2020-03-16 20:23:58 +08:00
// return false if the current patchset tag doesn't match the
2020-08-12 01:14:14 +08:00
// mainline publishable tag. i.e. ignore pg-9.5 builds
2020-02-15 06:02:23 +08:00
def isPatchsetPublishable() {
env.PATCHSET_TAG == env.PUBLISHABLE_TAG
2019-11-23 04:57:11 +08:00
}
2020-09-02 22:46:17 +08:00
def isPatchsetRetriggered() {
2020-09-03 22:31:09 +08:00
if(env.IS_AUTOMATIC_RETRIGGER == '1') {
return true
}
2020-09-02 22:46:17 +08:00
def userCause = currentBuild.getBuildCauses('com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritUserCause')
return userCause && userCause[0].shortDescription.contains('Retriggered')
}
2020-07-27 00:18:52 +08:00
def cleanupFn(status) {
2020-07-29 03:14:53 +08:00
ignoreBuildNeverStartedError {
2020-10-23 22:45:58 +08:00
execute 'bash/docker-cleanup.sh --allow-failure'
2020-07-27 00:18:52 +08:00
}
}
2020-07-29 03:14:53 +08:00
def postFn(status) {
2020-11-04 01:38:13 +08:00
def requestStartTime = System.currentTimeMillis()
2020-10-23 22:45:58 +08:00
node('master') {
2020-11-04 01:38:13 +08:00
def requestEndTime = System.currentTimeMillis()
reportToSplunk('node_request_time', [
'nodeName': 'master',
'nodeLabel': 'master',
'requestTime': requestEndTime - requestStartTime,
])
2020-10-23 22:45:58 +08:00
failureReport.publishReportFromArtifacts('Rspec Test Failures', "tmp/spec_failures/rspec/**/*")
failureReport.publishReportFromArtifacts('Selenium Test Failures', "tmp/spec_failures/selenium/**/*")
failureReport.submit()
}
2020-09-02 22:46:17 +08:00
if(status == 'FAILURE') {
maybeSlackSendFailure()
2020-09-03 22:31:09 +08:00
maybeRetrigger()
2020-09-02 22:46:17 +08:00
} else if(status == 'SUCCESS') {
maybeSlackSendSuccess()
}
}
2020-09-03 22:31:09 +08:00
def shouldPatchsetRetrigger() {
// NOTE: The IS_AUTOMATIC_RETRIGGER check is here to ensure that the parameter is properly defined for the triggering job.
// If it isn't, we have the risk of triggering this job over and over in an infinite loop.
return env.IS_AUTOMATIC_RETRIGGER == '0' && (
env.GERRIT_EVENT_TYPE == 'change-merged' ||
configuration.getBoolean('change-merged') && configuration.getBoolean('enable-automatic-retrigger', '0')
)
}
def maybeRetrigger() {
if(shouldPatchsetRetrigger() && !isPatchsetRetriggered()) {
def retriggerParams = currentBuild.rawBuild.getAction(ParametersAction).getParameters()
retriggerParams = retriggerParams.findAll { record ->
record.name != 'IS_AUTOMATIC_RETRIGGER'
}
retriggerParams << new StringParameterValue('IS_AUTOMATIC_RETRIGGER', "1")
build(job: env.JOB_NAME, parameters: retriggerParams, propagate: false, wait: false)
}
}
2020-09-02 22:46:17 +08:00
def maybeSlackSendFailure() {
if(configuration.isChangeMerged()) {
2020-07-29 03:14:53 +08:00
def branchSegment = env.GERRIT_BRANCH ? "[$env.GERRIT_BRANCH]" : ''
def authorSlackId = env.GERRIT_EVENT_ACCOUNT_EMAIL ? slackUserIdFromEmail(email: env.GERRIT_EVENT_ACCOUNT_EMAIL, botUser: true, tokenCredentialId: 'slack-user-id-lookup') : ''
def authorSlackMsg = authorSlackId ? "<@$authorSlackId>" : env.GERRIT_EVENT_ACCOUNT_NAME
2020-09-09 02:22:51 +08:00
def authorSegment = "Patchset <${env.GERRIT_CHANGE_URL}|#${env.GERRIT_CHANGE_NUMBER}> by ${authorSlackMsg} failed against ${branchSegment}"
2020-09-18 04:59:28 +08:00
def extra = "Please investigate the cause of the failure, and respond to this message with your diagnosis. If you need help, don't hesitate to tag @ oncall and our on call will assist in looking at the build. Further details of our post-merge failure process can be found at this <${configuration.getFailureWiki()}|link>. Thanks!"
2020-09-09 02:22:51 +08:00
2020-07-29 03:14:53 +08:00
slackSend(
2020-09-02 05:02:38 +08:00
channel: getSlackChannel(),
2020-07-29 03:14:53 +08:00
color: 'danger',
2020-09-09 02:22:51 +08:00
message: "${authorSegment}. Build <${env.BUILD_URL}|#${env.BUILD_NUMBER}>\n\n$extra"
2020-09-02 01:18:16 +08:00
)
}
}
2020-09-02 22:46:17 +08:00
def maybeSlackSendSuccess() {
if(configuration.isChangeMerged() && isPatchsetRetriggered()) {
slackSend(
channel: getSlackChannel(),
color: 'good',
message: "Patchset <${env.GERRIT_CHANGE_URL}|#${env.GERRIT_CHANGE_NUMBER}> succeeded on re-trigger. Build <${env.BUILD_URL}|#${env.BUILD_NUMBER}>"
)
}
}
2020-09-02 01:18:16 +08:00
2020-09-02 22:46:17 +08:00
def maybeSlackSendRetrigger() {
if(configuration.isChangeMerged() && isPatchsetRetriggered()) {
2020-09-02 01:18:16 +08:00
slackSend(
channel: getSlackChannel(),
color: 'warning',
2020-09-03 22:16:39 +08:00
message: "Patchset <${env.GERRIT_CHANGE_URL}|#${env.GERRIT_CHANGE_NUMBER}> by ${env.GERRIT_EVENT_ACCOUNT_EMAIL} has been re-triggered. Build <${env.BUILD_URL}|#${env.BUILD_NUMBER}>"
2020-07-29 03:14:53 +08:00
)
}
}
2020-10-14 02:25:33 +08:00
def slackSendCacheAvailable(blockName = '') {
def GIT_REV = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
slackSend(
channel: '#jenkins_cache_noisy',
color: 'good',
2020-10-15 01:02:50 +08:00
message: "Uploaded New Image\n\nBuild: <${env.BUILD_URL}|#${env.BUILD_NUMBER}>\nImage: ${blockName}\nRevision: ${GIT_REV}\nInstance: ${env.NODE_NAME}"
2020-10-14 02:25:33 +08:00
)
}
def slackSendCacheBuild(blockName = '', block) {
def buildStartTime = System.currentTimeMillis()
block()
def buildEndTime = System.currentTimeMillis()
def PARENT_GIT_REV = sh(script: 'git rev-parse HEAD^', returnStdout: true).trim()
slackSend(
channel: '#jenkins_cache_noisy',
2020-10-15 01:02:50 +08:00
message: "Built Image\n\nBuild: <${env.BUILD_URL}|#${env.BUILD_NUMBER}>\nImage: ${blockName}\nParent: ${PARENT_GIT_REV}\nDuration: ${buildEndTime - buildStartTime}ms\nInstance: ${env.NODE_NAME}"
2020-10-14 02:25:33 +08:00
)
}
2020-08-05 05:42:10 +08:00
// These functions are intentionally pinned to GERRIT_EVENT_TYPE == 'change-merged' to ensure that real post-merge
// builds always run correctly. We intentionally ignore overrides for version pins, docker image paths, etc when
// running real post-merge builds.
// =========
def getBuildImage() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? configuration.buildRegistryPathDefault() : configuration.buildRegistryPath()
}
def getPatchsetTag() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.patchsetDefault() : imageTag.patchset()
}
2020-09-24 22:28:35 +08:00
def getPublishableTag() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.publishableTagDefault() : imageTag.publishableTag()
}
2020-08-01 00:30:29 +08:00
def getPluginVersion(plugin) {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? 'master' : configuration.getString("pin-commit-$plugin", "master")
}
2020-09-02 05:02:38 +08:00
def getSlackChannel() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? '#canvas_builds' : '#devx-bots'
2020-08-05 05:42:10 +08:00
}
2020-09-02 05:02:38 +08:00
2020-09-30 01:17:57 +08:00
def getDependenciesMergeImage() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.dependenciesMergeImageDefault() : imageTag.dependenciesMergeImage()
}
def getDependenciesPatchsetImage() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.dependenciesPatchsetImageDefault() : imageTag.dependenciesPatchsetImage()
}
2020-10-01 00:55:16 +08:00
def getMergeTag() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.mergeTagDefault() : imageTag.mergeTag()
}
def getExternalTag() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? imageTag.externalTagDefault() : imageTag.externalTag()
}
2020-09-30 01:17:57 +08:00
def getDependenciesImage() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? configuration.dependenciesImageDefault() : configuration.dependenciesImage()
}
2020-11-02 23:39:02 +08:00
@groovy.transform.Field def CANVAS_BUILDS_REFSPEC_REGEX = /\[canvas\-builds\-refspec=(.+?)\]/
def getCanvasBuildsRefspec() {
def commitMessage = env.GERRIT_CHANGE_COMMIT_MESSAGE ? new String(env.GERRIT_CHANGE_COMMIT_MESSAGE.decodeBase64()) : null
if(env.GERRIT_EVENT_TYPE == 'change-merged' || !commitMessage || !(commitMessage =~ CANVAS_BUILDS_REFSPEC_REGEX).find()) {
return 'master'
}
return (commitMessage =~ CANVAS_BUILDS_REFSPEC_REGEX).findAll()[0][1]
}
2020-10-08 22:19:24 +08:00
def getCanvasLmsRefspec() {
return env.GERRIT_EVENT_TYPE == 'change-merged' ? configuration.canvasLmsRefspecDefault() : configuration.canvasLmsRefspec()
}
2020-08-05 05:42:10 +08:00
// =========
2020-11-02 23:39:02 +08:00
library "canvas-builds-library@${getCanvasBuildsRefspec()}"
2019-02-16 04:16:49 +08:00
pipeline {
2020-07-27 00:18:52 +08:00
agent none
2020-05-09 02:23:07 +08:00
options {
ansiColor('xterm')
timestamps()
}
2019-02-20 23:41:00 +08:00
2019-02-21 01:06:06 +08:00
environment {
2019-03-12 09:55:34 +08:00
GERRIT_PORT = '29418'
2019-02-21 01:22:05 +08:00
GERRIT_URL = "$GERRIT_HOST:$GERRIT_PORT"
2020-06-17 02:23:25 +08:00
BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
2020-08-05 05:42:10 +08:00
BUILD_IMAGE = getBuildImage()
2020-06-17 02:23:25 +08:00
POSTGRES = configuration.postgres()
2020-03-16 20:23:58 +08:00
POSTGRES_CLIENT = configuration.postgresClient()
2020-07-31 00:27:16 +08:00
SKIP_CACHE = configuration.skipCache()
2019-02-16 04:16:49 +08:00
2020-08-12 01:14:14 +08:00
// e.g. postgres-12-ruby-2.6
2020-07-22 10:06:57 +08:00
TAG_SUFFIX = imageTag.suffix()
2019-02-21 01:22:05 +08:00
2020-05-14 23:19:56 +08:00
2020-03-16 20:23:58 +08:00
// e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6
2020-08-05 05:42:10 +08:00
PATCHSET_TAG = getPatchsetTag()
2020-02-15 06:02:23 +08:00
2020-08-12 01:14:14 +08:00
// e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6
2020-09-24 22:28:35 +08:00
PUBLISHABLE_TAG = getPublishableTag()
2020-02-15 06:02:23 +08:00
// e.g. canvas-lms:master when not on another branch
2020-10-01 00:55:16 +08:00
MERGE_TAG = getMergeTag();
2020-04-04 04:05:52 +08:00
// e.g. canvas-lms:01.123456.78; this is for consumers like Portal 2 who want to build a patchset
2020-10-01 00:55:16 +08:00
EXTERNAL_TAG = getExternalTag();
2020-03-16 20:23:58 +08:00
ALPINE_MIRROR = configuration.alpineMirror()
NODE = configuration.node()
RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
2020-06-27 03:37:32 +08:00
2020-10-20 02:44:43 +08:00
PREMERGE_CACHE_IMAGE = "$BUILD_IMAGE-pre-merge-cache:$GERRIT_BRANCH"
POSTMERGE_CACHE_IMAGE = "$BUILD_IMAGE-post-merge-cache:$GERRIT_BRANCH"
2020-08-10 03:30:45 +08:00
2020-06-27 03:37:32 +08:00
CASSANDRA_IMAGE_TAG=imageTag.cassandra()
DYNAMODB_IMAGE_TAG=imageTag.dynamodb()
POSTGRES_IMAGE_TAG=imageTag.postgres()
2020-08-15 00:13:24 +08:00
// This is primarily for the plugin build
// for testing canvas-lms changes against plugin repo changes
2020-11-02 23:39:02 +08:00
CANVAS_BUILDS_REFSPEC = getCanvasBuildsRefspec()
2020-10-08 22:19:24 +08:00
CANVAS_LMS_REFSPEC = getCanvasLmsRefspec()
2020-08-28 20:08:22 +08:00
DOCKER_WORKDIR = getDockerWorkDir()
LOCAL_WORKDIR = getLocalWorkDir()
2020-02-15 06:02:23 +08:00
}
2019-12-10 02:37:05 +08:00
2020-02-15 06:02:23 +08:00
stages {
2020-07-27 00:18:52 +08:00
stage('Environment') {
2019-03-12 09:55:34 +08:00
steps {
2020-07-27 00:18:52 +08:00
script {
2020-08-05 05:42:10 +08:00
// Ensure that all build flags are compatible.
if(configuration.getBoolean('change-merged') && configuration.isValueDefault('build-registry-path')) {
error "Manually triggering the change-merged build path must be combined with a custom build-registry-path"
return
}
2020-09-02 22:46:17 +08:00
maybeSlackSendRetrigger()
2020-09-02 01:18:16 +08:00
2020-07-31 05:44:56 +08:00
// Use a nospot instance for now to avoid really bad UX. Jenkins currently will
// wait for the current steps to complete (even wait to spin up a node), causing
// extremely long wait times for a restart. Investigation in DE-166 / DE-158.
protectedNode('canvas-docker-nospot', { status -> cleanupFn(status) }, { status -> postFn(status) }) {
2020-10-07 23:23:23 +08:00
timedStage('Setup') {
2020-07-27 00:18:52 +08:00
timeout(time: 5) {
2020-10-20 00:26:25 +08:00
echo "Cleaning Workspace From Previous Runs"
sh 'ls -A1 | xargs rm -rf'
2020-10-13 21:17:08 +08:00
sh 'find .'
2020-07-27 00:18:52 +08:00
cleanAndSetup()
2020-10-21 23:24:29 +08:00
2020-10-23 00:21:11 +08:00
def refspecToCheckout = env.GERRIT_PROJECT == "canvas-lms" ? env.GERRIT_REFSPEC : env.CANVAS_LMS_REFSPEC
2020-10-21 23:24:29 +08:00
checkout([
$class: 'GitSCM',
branches: [[name: 'FETCH_HEAD']],
doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'CloneOption', depth: 100, honorRefspec: true, noTags: true, shallow: true]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: '44aa91d6-ab24-498a-b2b4-911bcb17cc35',
name: 'origin',
refspec: refspecToCheckout,
url: "ssh://gerrit.instructure.com:29418/canvas-lms.git"
]]
])
2020-07-27 00:18:52 +08:00
2020-11-02 23:39:02 +08:00
buildParameters += string(name: 'CANVAS_BUILDS_REFSPEC', value: "${env.CANVAS_BUILDS_REFSPEC}")
2020-07-27 00:18:52 +08:00
buildParameters += string(name: 'PATCHSET_TAG', value: "${env.PATCHSET_TAG}")
buildParameters += string(name: 'POSTGRES', value: "${env.POSTGRES}")
buildParameters += string(name: 'RUBY', value: "${env.RUBY}")
2020-10-23 00:21:11 +08:00
if (currentBuild.projectName.contains("rails-6")) {
buildParameters += string(name: 'CANVAS_RAILS6_0', value: "${env.CANVAS_RAILS6_0}")
}
2020-11-02 23:35:53 +08:00
// If modifying any of our Jenkinsfiles set JENKINSFILE_REFSPEC for sub-builds to use Jenkinsfiles in
// the gerrit rather than master.
if(env.GERRIT_PROJECT == 'canvas-lms' && (sh(script: './build/new-jenkins/jenkinsfile_changes.sh', returnStatus: true) == 0)) {
buildParameters += string(name: 'JENKINSFILE_REFSPEC', value: "${env.GERRIT_REFSPEC}")
}
2020-10-23 00:21:11 +08:00
if (env.GERRIT_PROJECT != "canvas-lms") {
2020-07-27 00:18:52 +08:00
// the plugin builds require the canvas lms refspec to be different. so only
// set this refspec if the main build is requesting it to be set.
// NOTE: this is only being set in main-from-plugin build. so main-canvas wont run this.
buildParameters += string(name: 'CANVAS_LMS_REFSPEC', value: env.CANVAS_LMS_REFSPEC)
}
2020-03-21 10:56:24 +08:00
2020-07-27 00:18:52 +08:00
pullGerritRepo('gerrit_builder', 'master', '.')
gems = readFile('gerrit_builder/canvas-lms/config/plugins_list').split()
echo "Plugin list: ${gems}"
// fetch plugins
gems.each { gem ->
if (env.GERRIT_PROJECT == gem) {
/* this is the commit we're testing */
2020-08-28 20:08:22 +08:00
dir(env.LOCAL_WORKDIR) {
checkout([
$class: 'GitSCM',
branches: [[name: 'FETCH_HEAD']],
doGenerateSubmoduleConfigurations: false,
2020-10-21 23:24:29 +08:00
extensions: [[$class: 'CloneOption', depth: 1, honorRefspec: true, noTags: true, shallow: true]],
2020-08-28 20:08:22 +08:00
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: '44aa91d6-ab24-498a-b2b4-911bcb17cc35',
name: 'origin',
refspec: "$env.GERRIT_REFSPEC",
url: "ssh://$GERRIT_URL/${GERRIT_PROJECT}.git"
]]
])
}
2020-07-27 00:18:52 +08:00
} else {
2020-08-01 00:30:29 +08:00
pullGerritRepo(gem, getPluginVersion(gem), 'gems/plugins')
2020-07-27 00:18:52 +08:00
}
}
2020-08-01 00:30:29 +08:00
pullGerritRepo("qti_migration_tool", getPluginVersion('qti_migration_tool'), "vendor")
2020-07-27 00:18:52 +08:00
2020-08-28 20:08:22 +08:00
// Plugin builds using the checkout above will create this @tmp file, we need to remove it
sh(script: 'rm -vr gems/plugins/*@tmp', returnStatus: true)
2020-07-27 00:18:52 +08:00
sh 'mv -v gerrit_builder/canvas-lms/config/* config/'
sh 'rm -v config/cache_store.yml'
sh 'rm -vr gerrit_builder'
sh 'rm -v config/database.yml'
sh 'rm -v config/security.yml'
sh 'rm -v config/selenium.yml'
sh 'rm -v config/file_store.yml'
sh 'cp -v docker-compose/config/selenium.yml config/'
sh 'cp -vR docker-compose/config/new-jenkins/* config/'
sh 'cp -v config/delayed_jobs.yml.example config/delayed_jobs.yml'
sh 'cp -v config/domain.yml.example config/domain.yml'
sh 'cp -v config/external_migration.yml.example config/external_migration.yml'
sh 'cp -v config/outgoing_mail.yml.example config/outgoing_mail.yml'
2019-10-22 03:55:38 +08:00
}
2020-04-01 01:25:11 +08:00
}
2019-03-05 05:09:25 +08:00
2020-08-05 05:42:10 +08:00
if(!configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms' && !configuration.skipRebase()) {
2020-10-07 23:23:23 +08:00
timedStage('Rebase') {
2020-07-27 00:18:52 +08:00
timeout(time: 2) {
credentials.withGerritCredentials({ ->
sh '''#!/bin/bash
set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
GIT_SSH_COMMAND='ssh -i \"$SSH_KEY_PATH\" -l \"$SSH_USERNAME\"' \
2020-10-20 04:10:03 +08:00
git fetch --depth 100 --force --no-tags origin $GERRIT_BRANCH:origin/$GERRIT_BRANCH
if ! git merge-base HEAD origin/master; then
echo "Error: your branch is over 100 commits behind, please rebase your branch manually."
exit $exit_status
fi
2020-07-27 00:18:52 +08:00
git config user.name "$GERRIT_EVENT_ACCOUNT_NAME"
git config user.email "$GERRIT_EVENT_ACCOUNT_EMAIL"
2020-10-20 03:56:30 +08:00
# store exit_status inline to ensures the script doesn't exit here on failures
2020-10-20 04:10:03 +08:00
git rebase --preserve-merges --stat origin/$GERRIT_BRANCH; exit_status=$?
2020-07-27 00:18:52 +08:00
if [ $exit_status != 0 ]; then
echo "Warning: Rebase couldn't resolve changes automatically, please resolve these conflicts locally."
git rebase --abort
exit $exit_status
fi
'''
})
2020-03-17 01:10:43 +08:00
}
2020-04-04 04:05:52 +08:00
}
2020-01-03 00:39:27 +08:00
}
2019-07-29 23:55:07 +08:00
2020-10-07 23:23:23 +08:00
timedStage('Build Docker Image') {
2020-07-27 00:18:52 +08:00
timeout(time: 30) {
skipIfPreviouslySuccessful('docker-build-and-push') {
2020-08-05 05:42:10 +08:00
if (!configuration.isChangeMerged() && configuration.skipDockerBuild()) {
2020-07-27 00:18:52 +08:00
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $MERGE_TAG'
sh 'docker tag $MERGE_TAG $PATCHSET_TAG'
} else {
2020-10-14 02:25:33 +08:00
slackSendCacheBuild(configuration.isChangeMerged() ? 'post-merge' : 'pre-merge') {
withEnv([
2020-10-20 02:44:43 +08:00
"CACHE_TAG=${configuration.isChangeMerged() ? env.POSTMERGE_CACHE_IMAGE : env.PREMERGE_CACHE_IMAGE}",
2020-10-22 11:17:14 +08:00
"COMPILE_ADDITIONAL_ASSETS=${configuration.isChangeMerged() ? 1 : 0}",
2020-10-14 02:25:33 +08:00
"JS_BUILD_NO_UGLIFY=${configuration.isChangeMerged() ? 0 : 1}"
]) {
sh "build/new-jenkins/docker-build.sh $PATCHSET_TAG"
}
2020-07-25 01:21:26 +08:00
}
2020-07-27 00:18:52 +08:00
}
2020-10-20 02:44:43 +08:00
2020-07-27 00:18:52 +08:00
sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $PATCHSET_TAG"
2020-10-20 02:44:43 +08:00
2020-07-27 00:18:52 +08:00
if (isPatchsetPublishable()) {
sh 'docker tag $PATCHSET_TAG $EXTERNAL_TAG'
sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $EXTERNAL_TAG'
2020-03-10 00:55:12 +08:00
}
2020-10-20 02:44:43 +08:00
if (configuration.isChangeMerged()) {
sh 'docker tag $PATCHSET_TAG $POSTMERGE_CACHE_IMAGE'
sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $POSTMERGE_CACHE_IMAGE'
slackSendCacheAvailable('post-merge')
}
2020-03-10 00:55:12 +08:00
}
}
2019-11-06 02:02:37 +08:00
}
2020-06-24 10:33:02 +08:00
2020-10-09 00:56:59 +08:00
timedStage('Run Migrations') {
2020-06-24 10:33:02 +08:00
timeout(time: 10) {
2020-10-09 02:01:13 +08:00
withEnv([
"COMPOSE_FILE=docker-compose.new-jenkins.yml",
"POSTGRES_PASSWORD=sekret"
]) {
migrations.runMigrations()
sh 'docker-compose down --remove-orphans'
2020-06-24 10:33:02 +08:00
}
}
}
2020-07-27 00:18:52 +08:00
stage('Parallel Run Tests') {
2020-06-24 10:33:02 +08:00
withEnv([
"CASSANDRA_IMAGE_TAG=${migrations.cassandraTag()}",
"DYNAMODB_IMAGE_TAG=${migrations.dynamodbTag()}",
"POSTGRES_IMAGE_TAG=${migrations.postgresTag()}"
]) {
2020-10-07 22:36:43 +08:00
def stages = [:]
2020-08-25 22:51:49 +08:00
2020-10-20 09:44:21 +08:00
if (configuration.isChangeMerged()) {
2020-08-25 22:51:49 +08:00
echo 'adding Build Docker Image Cache'
stages['Build Docker Image Cache'] = {
skipIfPreviouslySuccessful("build-docker-cache") {
withEnv([
2020-10-20 02:44:43 +08:00
"CACHE_TAG=${env.PREMERGE_CACHE_IMAGE}",
2020-10-22 11:17:14 +08:00
"COMPILE_ADDITIONAL_ASSETS=0",
2020-08-25 22:51:49 +08:00
"JS_BUILD_NO_UGLIFY=1"
]) {
2020-10-14 02:25:33 +08:00
slackSendCacheBuild('pre-merge') {
2020-10-20 02:44:43 +08:00
sh "build/new-jenkins/docker-build.sh $PREMERGE_CACHE_IMAGE"
2020-10-14 02:25:33 +08:00
}
2020-10-20 02:44:43 +08:00
sh "build/new-jenkins/docker-with-flakey-network-protection.sh push $PREMERGE_CACHE_IMAGE"
2020-10-14 02:25:33 +08:00
slackSendCacheAvailable('pre-merge')
2020-08-25 22:51:49 +08:00
}
}
}
}
2020-11-04 05:51:41 +08:00
if (!configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms') {
2020-10-07 22:36:43 +08:00
echo 'adding Linters'
2020-10-07 23:23:23 +08:00
timedStage('Linters', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("linters") {
credentials.withGerritCredentials {
sh 'build/new-jenkins/linters/run-gergich.sh'
}
if (env.MASTER_BOUNCER_RUN == '1' && !configuration.isChangeMerged()) {
credentials.withMasterBouncerCredentials {
sh 'build/new-jenkins/linters/run-master-bouncer.sh'
}
2020-07-27 00:18:52 +08:00
}
}
2020-10-07 23:23:23 +08:00
})
2020-07-27 00:18:52 +08:00
}
2020-04-11 03:38:35 +08:00
2020-10-07 22:36:43 +08:00
echo 'adding Consumer Smoke Test'
2020-10-07 23:23:23 +08:00
timedStage('Consumer Smoke Test', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("consumer-smoke-test") {
sh 'build/new-jenkins/consumer-smoke-test.sh'
}
2020-10-07 23:23:23 +08:00
})
2019-11-24 04:34:22 +08:00
2020-10-07 22:36:43 +08:00
echo 'adding Vendored Gems'
2020-10-07 23:23:23 +08:00
timedStage('Vendored Gems', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("vendored-gems") {
wrapBuildExecution('/Canvas/test-suites/vendored-gems', buildParameters + [
string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
], true, "")
}
2020-10-07 23:23:23 +08:00
})
2020-07-02 21:15:23 +08:00
2020-10-07 22:36:43 +08:00
echo 'adding Javascript (Jest)'
2020-10-07 23:23:23 +08:00
timedStage('Javascript (Jest)', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("javascript_jest") {
wrapBuildExecution('/Canvas/test-suites/JS', buildParameters + [
string(name: 'TEST_SUITE', value: "jest"),
], true, "testReport")
}
2020-10-07 23:23:23 +08:00
})
2019-11-27 00:36:22 +08:00
2020-10-07 22:36:43 +08:00
echo 'adding Javascript (Karma)'
2020-10-07 23:23:23 +08:00
timedStage('Javascript (Karma)', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("javascript_karma") {
wrapBuildExecution('/Canvas/test-suites/JS', buildParameters + [
string(name: 'TEST_SUITE', value: "karma"),
], true, "testReport")
}
2020-10-07 23:23:23 +08:00
})
2019-12-05 03:57:36 +08:00
2020-10-07 22:36:43 +08:00
echo 'adding Contract Tests'
2020-10-07 23:23:23 +08:00
timedStage('Contract Tests', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("contract-tests") {
wrapBuildExecution('/Canvas/test-suites/contract-tests', buildParameters + [
string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
], true, "")
}
2020-10-07 23:23:23 +08:00
})
2019-11-16 01:49:57 +08:00
2020-10-07 22:36:43 +08:00
if (sh(script: 'build/new-jenkins/check-for-migrations.sh', returnStatus: true) == 0) {
echo 'adding CDC Schema check'
2020-10-07 23:23:23 +08:00
timedStage('CDC Schema Check', stages, {
2020-10-07 22:36:43 +08:00
build job: '../Canvas/cdc-event-transformer-master', parameters: [
string(name: 'CANVAS_LMS_IMAGE_PATH', value: "${env.PATCHSET_TAG}")
]
2020-10-07 23:23:23 +08:00
})
2020-10-07 22:36:43 +08:00
}
else {
echo 'no migrations added, skipping CDC Schema check'
2020-06-23 14:16:12 +08:00
}
2020-10-07 22:36:43 +08:00
if (
!configuration.isChangeMerged() &&
(
dir(env.LOCAL_WORKDIR){ (sh(script: '${WORKSPACE}/build/new-jenkins/spec-changes.sh', returnStatus: true) == 0) } ||
configuration.forceFailureFSC() == '1'
)
) {
echo 'adding Flakey Spec Catcher'
2020-10-07 23:23:23 +08:00
timedStage('Flakey Spec Catcher', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("flakey-spec-catcher") {
def propagate = configuration.fscPropagate()
echo "fsc propagation: $propagate"
wrapBuildExecution('/Canvas/test-suites/flakey-spec-catcher', buildParameters + [
string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
], propagate, "")
}
2020-10-07 23:23:23 +08:00
})
2020-07-27 00:18:52 +08:00
}
2019-03-20 23:16:16 +08:00
2020-10-07 22:36:43 +08:00
if(env.GERRIT_PROJECT == 'canvas-lms' && (sh(script: 'build/new-jenkins/docker-dev-changes.sh', returnStatus: true) == 0)) {
echo 'adding Local Docker Dev Build'
2020-10-07 23:23:23 +08:00
timedStage('Local Docker Dev Build', stages, {
2020-10-07 22:36:43 +08:00
skipIfPreviouslySuccessful("local-docker-dev-smoke") {
wrapBuildExecution('/Canvas/test-suites/local-docker-dev-smoke', buildParameters, true, "")
}
2020-10-07 23:23:23 +08:00
})
2020-08-22 03:42:49 +08:00
}
2020-11-03 03:25:23 +08:00
if(configuration.isChangeMerged()) {
timedStage('Dependency Check', stages, {
snyk("canvas-lms:ruby", "Gemfile.lock", "$PATCHSET_TAG")
})
}
2020-10-07 22:36:43 +08:00
def distribution = load 'build/new-jenkins/groovy/distribution.groovy'
distribution.stashBuildScripts()
2020-07-27 00:18:52 +08:00
2020-10-07 22:36:43 +08:00
distribution.addRSpecSuites(stages)
distribution.addSeleniumSuites(stages)
2020-07-27 00:18:52 +08:00
2020-10-07 22:36:43 +08:00
parallel(stages)
}
2020-07-24 23:34:44 +08:00
}
2020-03-14 01:46:34 +08:00
2020-08-05 05:42:10 +08:00
if(configuration.isChangeMerged() && isPatchsetPublishable()) {
2020-10-07 23:23:23 +08:00
timedStage('Publish Image on Merge') {
2020-07-27 00:18:52 +08:00
timeout(time: 10) {
// Retriggers won't have an image to tag/push, pull that
// image if doesn't exist. If image is not found it will
// return NULL
if (!sh (script: 'docker images -q $PATCHSET_TAG')) {
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $PATCHSET_TAG'
}
2020-02-27 00:25:50 +08:00
2020-07-27 00:18:52 +08:00
// publish canvas-lms:$GERRIT_BRANCH (i.e. canvas-lms:master)
sh 'docker tag $PUBLISHABLE_TAG $MERGE_TAG'
2020-09-10 07:25:35 +08:00
def GIT_REV = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
sh "docker tag \$PUBLISHABLE_TAG \$BUILD_IMAGE:${GIT_REV}"
2020-07-27 00:18:52 +08:00
// push *all* canvas-lms images (i.e. all canvas-lms prefixed tags)
2020-09-10 07:25:35 +08:00
sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $BUILD_IMAGE'
2020-07-27 00:18:52 +08:00
}
}
}
2020-02-27 00:25:50 +08:00
2020-10-07 23:23:23 +08:00
timedStage('Mark Build as Successful') {
2020-07-27 00:18:52 +08:00
successes.markBuildAsSuccessful()
}
2020-08-22 03:42:49 +08:00
}//protectedNode
}//script
}//steps
}//environment
}//stages
}//pipline
