split linter stages into own node

refs DE-670

For the EKS transition, we need to ensure that the Linters stage doesn’t run on the EC2 builder node and instead properly uses EKS containers. As part of this work, transition linter stages into their own node as a sibling stage to Builder.

Test Plan
1. Linters run correctly
2. allow-stages directive works with Linters
3. allow-stages directive works without Linters

Change-Id: Ieae121ae3e79b5e30864f9060042e4453a8093e6
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/264569
Reviewed-by: Andrea Cirulli <andrea.cirulli@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>

Jenkinsfile

@@ -19,6 +19,7 @@
  */
 def FILES_CHANGED_STAGE = 'Detect Files Changed'
 def JS_BUILD_IMAGE_STAGE = 'Javascript (Build Image)'
+def LINTERS_BUILD_IMAGE_STAGE = 'Linters (Build Image)'
 def RUN_MIGRATIONS_STAGE = 'Run Migrations'
 
 def buildParameters = [
@@ -417,29 +418,6 @@ pipeline {
               extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { _, buildConfig ->
                 def stages = [:]
 
-                def linterHooks = [
-                  onNodeAcquired: lintersStage.&setupNode,
-                  onNodeReleasing: lintersStage.&tearDownNode,
-                ]
-
-                extendedStage('Linters')
-                  .hooks(linterHooks)
-                  .required(!configuration.isChangeMerged())
-                  .queue(stages, {
-                    def nestedStages = [:]
-
-                    extendedStage('Linters - Run Tests - Code').queue(nestedStages, lintersStage.&codeStage)
-                    extendedStage('Linters - Run Tests - Master Bouncer')
-                      .required(env.MASTER_BOUNCER_RUN == '1' && !configuration.isChangeMerged())
-                      .queue(nestedStages, lintersStage.&masterBouncerStage)
-                    extendedStage('Linters - Run Tests - Webpack').queue(nestedStages, lintersStage.&webpackStage)
-                    extendedStage('Linters - Run Tests - Yarn')
-                      .required(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(['package.json', 'yarn.lock'], 'HEAD^'))
-                      .queue(nestedStages, lintersStage.&yarnStage)
-
-                    parallel(nestedStages)
-                  })
-
                 extendedStage('Consumer Smoke Test').queue(stages) {
                   sh 'build/new-jenkins/consumer-smoke-test.sh'
                 }
@@ -447,6 +425,9 @@ pipeline {
                 extendedStage(JS_BUILD_IMAGE_STAGE)
                   .queue(stages, buildDockerImageStage.&jsImage)
 
+                extendedStage(LINTERS_BUILD_IMAGE_STAGE)
+                  .queue(stages, buildDockerImageStage.&lintersImage)
+
                 extendedStage('Dependency Check')
                   .required(configuration.isChangeMerged())
                   .queue(stages, { dependencyCheckStage() })
@@ -493,6 +474,32 @@ pipeline {
               parallel(nestedStages)
             }
 
+            extendedStage('Linters (Waiting for Dependencies)').obeysAllowStages(false).waitsFor(LINTERS_BUILD_IMAGE_STAGE, 'Builder').queue(rootStages) {
+              def linterHooks = [
+                onNodeAcquired: lintersStage.&setupNode,
+                onNodeReleasing: lintersStage.&tearDownNode,
+              ]
+
+              extendedStage('Linters')
+                .hooks(linterHooks)
+                .nodeRequirements(label: 'canvas-docker', podTemplate: libraryResource('/pod_templates/docker_base.yml'), container: 'docker')
+                .required(!configuration.isChangeMerged())
+                .execute {
+                  def nestedStages = [:]
+
+                  extendedStage('Linters - Run Tests - Code').queue(nestedStages, lintersStage.&codeStage)
+                  extendedStage('Linters - Run Tests - Master Bouncer')
+                    .required(env.MASTER_BOUNCER_RUN == '1' && !configuration.isChangeMerged())
+                    .queue(nestedStages, lintersStage.&masterBouncerStage)
+                  extendedStage('Linters - Run Tests - Webpack').queue(nestedStages, lintersStage.&webpackStage)
+                  extendedStage('Linters - Run Tests - Yarn')
+                    .required(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(['package.json', 'yarn.lock'], 'HEAD^'))
+                    .queue(nestedStages, lintersStage.&yarnStage)
+
+                  parallel(nestedStages)
+                }
+            }
+
             extendedStage("${RUN_MIGRATIONS_STAGE} (Waiting for Dependencies)").obeysAllowStages(false).waitsFor(RUN_MIGRATIONS_STAGE, 'Builder').queue(rootStages) { _, buildConfig ->
               def nestedStages = [:]
 

build/new-jenkins/library/vars/buildDockerImageStage.groovy

@@ -109,6 +109,13 @@ def jsImage() {
   }
 }
 
+def lintersImage() {
+  credentials.withStarlordDockerLogin {
+    sh './build/new-jenkins/linters/docker-build.sh $LINTERS_RUNNER_IMAGE'
+    sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $LINTERS_RUNNER_PREFIX'
+  }
+}
+
 def premergeCacheImage() {
   credentials.withStarlordDockerLogin {
     withEnv([

build/new-jenkins/library/vars/distribution.groovy

@@ -42,25 +42,23 @@ def appendStagesAsBuildNodes(nodes,
     def timeStart = new Date()
     extendedStage(stageName).nodeRequirements(label: 'canvas-docker', podTemplate: libraryResource('/pod_templates/docker_base.yml'), container: 'docker').queue(nodes) {
       echo "Running on node ${env.NODE_NAME}"
-      def duration = TimeCategory.minus(new Date(), timeStart).toMilliseconds()
-      // make sure to unstash
-      unstash name: 'build-dir'
-      unstash name: 'build-docker-compose'
+
+      unstashBuildScripts()
       stageBlock(index)
     }
   }
-                             }
+}
 
-/**
- * use this in combination with appendStagesAsBuildNodes. this will
- * stash the required files for running biulds that only require
- * the build scripts
- */
 def stashBuildScripts() {
   stash name: 'build-dir', includes: 'build/**/*'
   stash name: 'build-docker-compose', includes: 'docker-compose.*.yml'
 }
 
+def unstashBuildScripts() {
+  unstash name: 'build-dir'
+  unstash name: 'build-docker-compose'
+}
+
 /**
  * common helper for adding rspec tests to be ran
  */

build/new-jenkins/library/vars/lintersStage.groovy

@@ -46,12 +46,9 @@ def _getDockerInputs() {
 }
 
 def setupNode() {
-  credentials.withStarlordDockerLogin {
-    sh './build/new-jenkins/linters/docker-build.sh $LINTERS_RUNNER_IMAGE'
-    sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $LINTERS_RUNNER_PREFIX'
+  distribution.unstashBuildScripts()
 
-    sh "docker volume create $dockerVolumeName"
-  }
+  sh "docker volume create $dockerVolumeName"
 }
 
 def tearDownNode() {