114 lines
4.6 KiB
Groovy
114 lines
4.6 KiB
Groovy
#!/usr/bin/env groovy
|
|
|
|
/*
|
|
* Copyright (C) 2019 - present Instructure, Inc.
|
|
*
|
|
* This file is part of Canvas.
|
|
*
|
|
* Canvas is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU Affero General Public License as published by the Free
|
|
* Software Foundation, version 3 of the License.
|
|
*
|
|
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
|
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
|
* details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License along
|
|
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
|
|
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
|
|
|
|
pipeline {
|
|
agent none
|
|
options {
|
|
ansiColor('xterm')
|
|
timeout(time: 15)
|
|
timestamps()
|
|
}
|
|
|
|
environment {
|
|
BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
|
|
DISABLE_SPRING = 'true'
|
|
COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins.vendored-gems.yml'
|
|
POSTGRES_PASSWORD = 'sekret'
|
|
}
|
|
|
|
stages {
|
|
stage('Environment') {
|
|
steps {
|
|
script {
|
|
def stageHooks = [
|
|
onNodeAcquired: vendoredGemsStage.setupNode(),
|
|
]
|
|
|
|
extendedStage('Runner').hooks(stageHooks).nodeRequirements(label: 'canvas-docker').obeysAllowStages(false).execute {
|
|
extendedStage('Vendored Gems').obeysAllowStages(false).timeout(10).execute {
|
|
sh 'build/new-jenkins/test-gems.sh'
|
|
}
|
|
|
|
extendedStage('Engines').obeysAllowStages(false).timeout(10).execute {
|
|
sh 'build/new-jenkins/test-engines.sh'
|
|
}
|
|
|
|
extendedStage('Pulsar').obeysAllowStages(false).required(false).timeout(10).execute {
|
|
withEnv([
|
|
'COMPOSE_FILE=docker-compose.new-jenkins.yml:docker-compose.new-jenkins.vendored-gems.yml:docker-compose.new-jenkins.pulsar.yml'
|
|
]) {
|
|
int retries = 0
|
|
int retryLimit = 4
|
|
Boolean stillRunning = true
|
|
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull starlord.inscloudgate.net/jenkins/pulsar:latest'
|
|
while (stillRunning && retries < retryLimit) {
|
|
try {
|
|
// sometimes the pulsar container kills itself on boot
|
|
// if it can't talk to it's own internal zookeeper.
|
|
// We haven't found out WHY yet (working on it),
|
|
// but we want the retry loop to be as fast as possible.
|
|
// If the admin script exits, we'll completely kill
|
|
// the container in the catch block and try again a limited
|
|
// number of times. The bootstrap script only contains
|
|
// a very limited number of retries because the pulsar client has exponential backoff
|
|
// baked in already so waiting many more times is just wasted time.
|
|
|
|
sh '''
|
|
docker-compose run pulsar-admin ./scripts/bootstrap.sh
|
|
docker-compose run canvas bash -c "cp ./config/overrides/dynamic_settings.pulsar.yml ./config/dynamic_settings.yml && ./build/new-jenkins/rspec-pulsar.sh"
|
|
'''
|
|
stillRunning = false
|
|
/* groovylint-disable-next-line CatchException */
|
|
} catch (Exception ex) {
|
|
// want to be able to debug in build output what
|
|
// made the pulsar container fail.
|
|
sh 'docker-compose logs pulsar'
|
|
|
|
// kinda hacky, but this will return a non-0 exit code if we did NOT find this string
|
|
// which means we don't think we can retry, so just let it die.
|
|
sh 'docker-compose logs pulsar | grep -E "zoo keeper disconnected|Failed to get Partitioned Metadata"'
|
|
|
|
reportToSplunk('pulsar_retrying')
|
|
sh '''
|
|
docker-compose stop pulsar
|
|
docker-compose rm -f pulsar
|
|
'''
|
|
retries += 1
|
|
if (retries >= retryLimit) {
|
|
sh '''
|
|
echo \"*********PULSAR LOGS*********\"
|
|
docker-compose logs pulsar
|
|
'''
|
|
reportToSplunk('pulsar_failed')
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|