diff --git a/Jenkinsfile b/Jenkinsfile index 2631e3d48ec..01d04d0ca38 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,6 +19,7 @@ */ def FILES_CHANGED_STAGE = 'Detect Files Changed' def JS_BUILD_IMAGE_STAGE = 'Javascript (Build Image)' +def LINTERS_BUILD_IMAGE_STAGE = 'Linters (Build Image)' def RUN_MIGRATIONS_STAGE = 'Run Migrations' def buildParameters = [ @@ -417,29 +418,6 @@ pipeline { extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { _, buildConfig -> def stages = [:] - def linterHooks = [ - onNodeAcquired: lintersStage.&setupNode, - onNodeReleasing: lintersStage.&tearDownNode, - ] - - extendedStage('Linters') - .hooks(linterHooks) - .required(!configuration.isChangeMerged()) - .queue(stages, { - def nestedStages = [:] - - extendedStage('Linters - Run Tests - Code').queue(nestedStages, lintersStage.&codeStage) - extendedStage('Linters - Run Tests - Master Bouncer') - .required(env.MASTER_BOUNCER_RUN == '1' && !configuration.isChangeMerged()) - .queue(nestedStages, lintersStage.&masterBouncerStage) - extendedStage('Linters - Run Tests - Webpack').queue(nestedStages, lintersStage.&webpackStage) - extendedStage('Linters - Run Tests - Yarn') - .required(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(['package.json', 'yarn.lock'], 'HEAD^')) - .queue(nestedStages, lintersStage.&yarnStage) - - parallel(nestedStages) - }) - extendedStage('Consumer Smoke Test').queue(stages) { sh 'build/new-jenkins/consumer-smoke-test.sh' } @@ -447,6 +425,9 @@ pipeline { extendedStage(JS_BUILD_IMAGE_STAGE) .queue(stages, buildDockerImageStage.&jsImage) + extendedStage(LINTERS_BUILD_IMAGE_STAGE) + .queue(stages, buildDockerImageStage.&lintersImage) + extendedStage('Dependency Check') .required(configuration.isChangeMerged()) .queue(stages, { dependencyCheckStage() }) @@ -493,6 +474,32 @@ pipeline { parallel(nestedStages) } + extendedStage('Linters (Waiting for Dependencies)').obeysAllowStages(false).waitsFor(LINTERS_BUILD_IMAGE_STAGE, 'Builder').queue(rootStages) { + def linterHooks = [ + onNodeAcquired: lintersStage.&setupNode, + onNodeReleasing: lintersStage.&tearDownNode, + ] + + extendedStage('Linters') + .hooks(linterHooks) + .nodeRequirements(label: 'canvas-docker', podTemplate: libraryResource('/pod_templates/docker_base.yml'), container: 'docker') + .required(!configuration.isChangeMerged()) + .execute { + def nestedStages = [:] + + extendedStage('Linters - Run Tests - Code').queue(nestedStages, lintersStage.&codeStage) + extendedStage('Linters - Run Tests - Master Bouncer') + .required(env.MASTER_BOUNCER_RUN == '1' && !configuration.isChangeMerged()) + .queue(nestedStages, lintersStage.&masterBouncerStage) + extendedStage('Linters - Run Tests - Webpack').queue(nestedStages, lintersStage.&webpackStage) + extendedStage('Linters - Run Tests - Yarn') + .required(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(['package.json', 'yarn.lock'], 'HEAD^')) + .queue(nestedStages, lintersStage.&yarnStage) + + parallel(nestedStages) + } + } + extendedStage("${RUN_MIGRATIONS_STAGE} (Waiting for Dependencies)").obeysAllowStages(false).waitsFor(RUN_MIGRATIONS_STAGE, 'Builder').queue(rootStages) { _, buildConfig -> def nestedStages = [:] diff --git a/build/new-jenkins/library/vars/buildDockerImageStage.groovy b/build/new-jenkins/library/vars/buildDockerImageStage.groovy index ef880e18c1d..640e275b73f 100644 --- a/build/new-jenkins/library/vars/buildDockerImageStage.groovy +++ b/build/new-jenkins/library/vars/buildDockerImageStage.groovy @@ -109,6 +109,13 @@ def jsImage() { } } +def lintersImage() { + credentials.withStarlordDockerLogin { + sh './build/new-jenkins/linters/docker-build.sh $LINTERS_RUNNER_IMAGE' + sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $LINTERS_RUNNER_PREFIX' + } +} + def premergeCacheImage() { credentials.withStarlordDockerLogin { withEnv([ diff --git a/build/new-jenkins/library/vars/distribution.groovy b/build/new-jenkins/library/vars/distribution.groovy index 38f8644b0a6..0043fec0480 100644 --- a/build/new-jenkins/library/vars/distribution.groovy +++ b/build/new-jenkins/library/vars/distribution.groovy @@ -42,25 +42,23 @@ def appendStagesAsBuildNodes(nodes, def timeStart = new Date() extendedStage(stageName).nodeRequirements(label: 'canvas-docker', podTemplate: libraryResource('/pod_templates/docker_base.yml'), container: 'docker').queue(nodes) { echo "Running on node ${env.NODE_NAME}" - def duration = TimeCategory.minus(new Date(), timeStart).toMilliseconds() - // make sure to unstash - unstash name: 'build-dir' - unstash name: 'build-docker-compose' + + unstashBuildScripts() stageBlock(index) } } - } +} -/** - * use this in combination with appendStagesAsBuildNodes. this will - * stash the required files for running biulds that only require - * the build scripts - */ def stashBuildScripts() { stash name: 'build-dir', includes: 'build/**/*' stash name: 'build-docker-compose', includes: 'docker-compose.*.yml' } +def unstashBuildScripts() { + unstash name: 'build-dir' + unstash name: 'build-docker-compose' +} + /** * common helper for adding rspec tests to be ran */ diff --git a/build/new-jenkins/library/vars/lintersStage.groovy b/build/new-jenkins/library/vars/lintersStage.groovy index 37362e26c9f..4a3eb3e137b 100644 --- a/build/new-jenkins/library/vars/lintersStage.groovy +++ b/build/new-jenkins/library/vars/lintersStage.groovy @@ -46,12 +46,9 @@ def _getDockerInputs() { } def setupNode() { - credentials.withStarlordDockerLogin { - sh './build/new-jenkins/linters/docker-build.sh $LINTERS_RUNNER_IMAGE' - sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $LINTERS_RUNNER_PREFIX' + distribution.unstashBuildScripts() - sh "docker volume create $dockerVolumeName" - } + sh "docker volume create $dockerVolumeName" } def tearDownNode() {