support pinning commonly modified gems to specific versions

Change-Id: I1bb63c1c1af144678236cd1a7ea63d1e10c666a4
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/286541
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>
Reviewed-by: James Butters <jbutters@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>

Dockerfile.jenkins-cache

@@ -27,6 +27,11 @@ RUN --mount=target=/tmp/src \
     -name "*.gemspec" -o \
     -name "gem_version.rb" -o \
     -name "version.rb" \
+  \) -exec cp --parents -rf {} /tmp/dst \; && \
+  find vendor -type d \( \
+    -not -name "qti_migration_tool" \
+    -maxdepth 1 \
+    -mindepth 1 \
   \) -exec cp --parents -rf {} /tmp/dst \;
 
 FROM starlord.inscloudgate.net/jenkins/busybox AS cache-helper-collect-yarn

Gemfile.d/app.rb

@@ -19,18 +19,26 @@
 
 # NOTE: Indented gems are meant to indicate transient dependencies of parent gems
 
+def gem_override(name, version)
+  if File.directory?("vendor/#{name}")
+    gem name, path: "vendor/#{name}"
+  else
+    gem name, version
+  end
+end
+
 if CANVAS_RAILS6_0
   gem "rails", "6.0.4.2"
     gem "tzinfo", "1.2.7"
-  gem "switchman", "2.2.2"
+  gem_override "switchman", "2.2.2"
     gem "guardrail", "2.0.1"
-  gem "switchman-inst-jobs", "3.2.10"
+  gem_override "switchman-inst-jobs", "3.2.10"
 else
   gem "rails", "6.1.4.6"
     gem "tzinfo", "2.0.4"
-  gem "switchman", "3.0.8"
+  gem_override "switchman", "3.0.8"
     gem "guardrail", "3.0.0"
-  gem "switchman-inst-jobs", "4.0.6"
+  gem_override "switchman-inst-jobs", "4.0.6"
 end
   gem "irb", "1.4.1"
   gem "open4", "1.3.4", require: false
@@ -88,7 +96,7 @@ gem "inst_statsd", "2.2.0"
   gem "statsd-ruby", "1.4.0", require: false
   gem "aroi", "0.0.7", require: false
   gem "dogstatsd-ruby", "4.8.2"
-gem "inst-jobs", "3.0.9"
+gem_override "inst-jobs", "3.0.9"
   gem "fugit", "1.4.0", require: false
     gem "et-orbi", "1.2.4", require: false
 gem "inst-jobs-autoscaling", "2.1.1"

Jenkinsfile

@@ -354,6 +354,8 @@ pipeline {
                   return
                 } else if (extendedStage.isAllowStagesFilterUsed() || extendedStage.isIgnoreStageResultsFilterUsed() || extendedStage.isSkipStagesFilterUsed()) {
                   gerrit.submitLintReview('-2', 'One or more build flags causes a subset of the build to be run')
+                } else if (setupStage.hasGemOverrides()) {
+                  gerrit.submitLintReview('-2', 'One or more build flags causes the build to be run against an unmerged gem version override')
                 } else {
                   gerrit.submitLintReview('0')
                 }
@@ -604,7 +606,7 @@ pipeline {
                       string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
                       string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
                       string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
-                      string(name: 'SKIP_CRYSTALBALL', value: "${env.SKIP_CRYSTALBALL}"),
+                      string(name: 'SKIP_CRYSTALBALL', value: "${env.SKIP_CRYSTALBALL || setupStage.hasGemOverrides()}"),
                       string(name: 'UPSTREAM_TAG', value: "${env.BUILD_TAG}"),
                       string(name: 'UPSTREAM', value: "${env.JOB_NAME}"),
                     ])

build/new-jenkins/library/vars/setupStage.groovy

@@ -16,6 +16,16 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+import groovy.transform.Field
+
+@Field final static OVERRIDABLE_GEMS = ['inst-jobs', 'switchman', 'switchman-inst-jobs']
+
+def hasGemOverrides() {
+  return OVERRIDABLE_GEMS.any { gem ->
+    return configuration.getString("pin-commit-$gem", "skip") != "skip"
+  }
+}
+
 def call() {
   def refspecToCheckout = env.GERRIT_PROJECT == 'canvas-lms' ? env.GERRIT_REFSPEC : env.CANVAS_LMS_REFSPEC
   checkoutRepo('canvas-lms', refspecToCheckout, 100)
@@ -39,6 +49,12 @@ def call() {
     }
   }
 
+  OVERRIDABLE_GEMS.each { gem ->
+    if (configuration.getString("pin-commit-$gem", "skip") != "skip") {
+      pluginsToPull.add([name: gem, version: _getPluginVersion(gem), target: "vendor/$gem"])
+    }
+  }
+
   if (env.GERRIT_PROJECT != 'qti_migration_tool') {
     pluginsToPull.add([name: 'qti_migration_tool', version: _getPluginVersion('qti_migration_tool'), target: 'vendor/qti_migration_tool'])
   }