split external package installation from gem installation

refs DE-545

When apt-get tries to pull a package from a server that is down or undergoing maintenance, builds that modify Gemfile can fail due to not being able to pull packages. By splitting the package installation into its own layer, external packages no longer need to be reinstalled when gems are updated.

Test Plan
1. Cache builds correctly from scratch, uploads all relevant images.
2. Cache is correctly reused when no gems are updated
3. Base cache image is reused when gems are updated, and all other images are rebuilt
4. Pre-merge build passes and uses cache as appropriate
5. Updating Dockerfile.jenkins re-builds base image

[build-registry-path=jenkins/canvas-lms/de-545-1]
[change-merged]

Change-Id: Ifd76064892817abb80dd0daebe8c2189c0338d78
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/279697
Reviewed-by: Andrea Cirulli <andrea.cirulli@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>

Dockerfile.jenkins

@@ -79,16 +79,3 @@ RUN if [ -e /var/lib/gems/$RUBY_MAJOR.0/gems/bundler-* ]; then BUNDLER_INSTALL="
 RUN npm install -g npm@latest && npm cache clean --force
 
 USER docker
-
-COPY --chown=docker:docker --from=local/cache-helper-collect-gems /tmp/dst ${APP_HOME}
-
-RUN set -eux; \
-  \
-  # set up bundle config options \
-  bundle config --global build.nokogiri --use-system-libraries \
-  && bundle config --global build.ffi --enable-system-libffi \
-  && mkdir -p \
-    /home/docker/.bundle \
-  # TODO: --without development \
-  && bundle install --jobs $(nproc) \
-  && rm -rf $GEM_HOME/cache

Dockerfile.jenkins.ruby-runner

@@ -0,0 +1,14 @@
+FROM local/base-runner
+
+COPY --chown=docker:docker --from=local/cache-helper-collect-gems /tmp/dst ${APP_HOME}
+
+RUN set -eux; \
+  \
+  # set up bundle config options \
+  bundle config --global build.nokogiri --use-system-libraries \
+  && bundle config --global build.ffi --enable-system-libffi \
+  && mkdir -p \
+    /home/docker/.bundle \
+  # TODO: --without development \
+  && bundle install --jobs $(nproc) \
+  && rm -rf $GEM_HOME/cache

Jenkinsfile

@@ -293,6 +293,7 @@ pipeline {
     NODE = configuration.node()
     RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
 
+    BASE_RUNNER_PREFIX = configuration.buildRegistryPath('base-runner')
     CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
     DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
     KARMA_BUILDER_PREFIX = configuration.buildRegistryPath('karma-builder')

build/Dockerfile.template

@@ -94,20 +94,6 @@ RUN npm install -g npm@latest && npm cache clean --force
 
 USER docker
 
-<% if jenkins? -%>
-COPY --chown=docker:docker --from=local/cache-helper-collect-gems /tmp/dst ${APP_HOME}
-
-RUN set -eux; \
-  \
-  # set up bundle config options \
-  bundle config --global build.nokogiri --use-system-libraries \
-  && bundle config --global build.ffi --enable-system-libffi \
-  && mkdir -p \
-    /home/docker/.bundle \
-  # TODO: --without development \
-  && bundle install --jobs $(nproc) \
-  && rm -rf $GEM_HOME/cache
-<% end -%>
 <% if development? -%>
 RUN set -eux; \
   mkdir -p \

build/new-jenkins/docker-build.sh

@@ -17,7 +17,8 @@ WORKSPACE=${WORKSPACE:-$(pwd)}
 #    layers not being used, even when their contents have not changed.
 
 # Images:
-# $RUBY_RUNNER_PREFIX: starlord.inscloudgate.net/jenkins/ruby-passenger + gems
+# $BASE_RUNNER_PREFIX: starlord.inscloudgate.net/jenkins/ruby-passenger + additional packages
+# $RUBY_RUNNER_PREFIX: $BASE_RUNNER_PREFIX + gems
 # $YARN_RUNNER_PREFIX: $RUBY_RUNNER_PREFIX + yarn
 # $WEBPACK_BUILDER_PREFIX: $YARN_RUNNER_PREFIX + compiled packages/
 # $WEBPACK_CACHE_PREFIX: $RUBY_RUNNER_PREFIX + final compiled assets
@@ -35,7 +36,7 @@ WORKSPACE=${WORKSPACE:-$(pwd)}
 # $WEBPACK_BUILDER_TAG: additional tag for the webpack-builder image
 #   - set to patchset unique ID for builds to reference without knowing about the hash ID
 
-export CACHE_VERSION="2021-07-12.1"
+export CACHE_VERSION="2021-11-29.1"
 
 source ./build/new-jenkins/docker-build-helpers.sh
 
@@ -52,7 +53,8 @@ YARN_CACHE_MD5=$(docker run local/cache-helper-collect-yarn sh -c "find /tmp/dst
 PACKAGES_CACHE_MD5=$(docker run local/cache-helper-collect-packages sh -c "find /tmp/dst -type f -exec md5sum {} \; | sort -k 2 | md5sum")
 WEBPACK_CACHE_MD5=$(docker run local/cache-helper-collect-webpack sh -c "find /tmp/dst -type f -exec md5sum {} \; | sort -k 2 | md5sum")
 
-RUBY_RUNNER_DOCKERFILE_MD5=$(cat Dockerfile.jenkins | md5sum)
+BASE_RUNNER_DOCKERFILE_MD5=$(cat Dockerfile.jenkins | md5sum)
+RUBY_RUNNER_DOCKERFILE_MD5=$(cat Dockerfile.jenkins.ruby-runner | md5sum)
 YARN_RUNNER_DOCKERFILE_MD5=$(cat Dockerfile.jenkins.yarn-runner | md5sum)
 WEBPACK_BUILDER_DOCKERFILE_MD5=$(cat Dockerfile.jenkins.webpack-builder | md5sum)
 WEBPACK_CACHE_DOCKERFILE_MD5=$(cat Dockerfile.jenkins.webpack-cache | md5sum)
@@ -61,7 +63,7 @@ WEBPACK_CACHE_DOCKERFILE_MD5=$(cat Dockerfile.jenkins.webpack-cache | md5sum)
 
 BASE_IMAGE_ID=$(docker images --filter=reference=starlord.inscloudgate.net/jenkins/ruby-passenger:$RUBY --format '{{.ID}}')
 
-RUBY_RUNNER_BUILD_ARGS=(
+BASE_RUNNER_BUILD_ARGS=(
   --build-arg CANVAS_RAILS6_0=${CANVAS_RAILS6_0:-1}
   --build-arg POSTGRES_CLIENT="$POSTGRES_CLIENT"
   --build-arg RUBY="$RUBY"
@@ -70,9 +72,13 @@ WEBPACK_CACHE_BUILD_ARGS=(
   --build-arg JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY"
   --build-arg RAILS_LOAD_ALL_LOCALES="$RAILS_LOAD_ALL_LOCALES"
 )
-RUBY_RUNNER_PARTS=(
+BASE_RUNNER_PARTS=(
   $BASE_IMAGE_ID
-  "${RUBY_RUNNER_BUILD_ARGS[@]}"
+  "${BASE_RUNNER_BUILD_ARGS[@]}"
+  $BASE_RUNNER_DOCKERFILE_MD5
+)
+RUBY_RUNNER_PARTS=(
+  "${BASE_RUNNER_PARTS[@]}"
   $RUBY_RUNNER_DOCKERFILE_MD5
   $RUBY_CACHE_MD5
 )
@@ -93,6 +99,7 @@ WEBPACK_CACHE_PARTS=(
   $WEBPACK_CACHE_MD5
 )
 
+declare -A BASE_RUNNER_TAGS; compute_tags "BASE_RUNNER_TAGS" $BASE_RUNNER_PREFIX ${BASE_RUNNER_PARTS[@]}
 declare -A RUBY_RUNNER_TAGS; compute_tags "RUBY_RUNNER_TAGS" $RUBY_RUNNER_PREFIX ${RUBY_RUNNER_PARTS[@]}
 declare -A YARN_RUNNER_TAGS; compute_tags "YARN_RUNNER_TAGS" $YARN_RUNNER_PREFIX ${YARN_RUNNER_PARTS[@]}
 declare -A WEBPACK_BUILDER_TAGS; compute_tags "WEBPACK_BUILDER_TAGS" $WEBPACK_BUILDER_PREFIX ${WEBPACK_BUILDER_PARTS[@]}
@@ -110,11 +117,26 @@ if [ -z "${WEBPACK_CACHE_SELECTED_TAG}" ]; then
       RUBY_RUNNER_SELECTED_TAG=""; pull_first_tag "RUBY_RUNNER_SELECTED_TAG" ${RUBY_RUNNER_TAGS[LOAD_TAG]} ${RUBY_RUNNER_TAGS[LOAD_FALLBACK_TAG]}
 
       if [ -z "${RUBY_RUNNER_SELECTED_TAG}" ]; then
+        BASE_RUNNER_SELECTED_TAG=""; pull_first_tag "BASE_RUNNER_SELECTED_TAG" ${BASE_RUNNER_TAGS[LOAD_TAG]} ${BASE_RUNNER_TAGS[LOAD_FALLBACK_TAG]}
+
+        if [ -z "${BASE_RUNNER_SELECTED_TAG}" ]; then
           docker build \
-          "${RUBY_RUNNER_BUILD_ARGS[@]}" \
+            "${BASE_RUNNER_BUILD_ARGS[@]}" \
-          --tag "${RUBY_RUNNER_TAGS[SAVE_TAG]}" \
+            --tag "${BASE_RUNNER_TAGS[SAVE_TAG]}" \
             - < Dockerfile.jenkins
 
+          BASE_RUNNER_SELECTED_TAG=${BASE_RUNNER_TAGS[SAVE_TAG]}
+
+          add_log "built ${BASE_RUNNER_SELECTED_TAG}"
+        fi
+
+        tag_many $BASE_RUNNER_SELECTED_TAG local/base-runner ${BASE_RUNNER_TAGS[SAVE_TAG]}
+
+        docker build \
+          --label "BASE_RUNNER_SELECTED_TAG=$BASE_RUNNER_SELECTED_TAG" \
+          --tag "${RUBY_RUNNER_TAGS[SAVE_TAG]}" \
+          - < Dockerfile.jenkins.ruby-runner
+
         RUBY_RUNNER_SELECTED_TAG=${RUBY_RUNNER_TAGS[SAVE_TAG]}
 
         add_log "built ${RUBY_RUNNER_SELECTED_TAG}"

build/new-jenkins/library/vars/buildDockerImageStage.groovy

@@ -119,6 +119,7 @@ def lintersImage() {
 def premergeCacheImage() {
   credentials.withStarlordCredentials {
     withEnv([
+      "BASE_RUNNER_PREFIX=${env.BASE_RUNNER_PREFIX}",
       "CACHE_LOAD_SCOPE=${env.IMAGE_CACHE_MERGE_SCOPE}",
       "CACHE_LOAD_FALLBACK_SCOPE=${env.IMAGE_CACHE_BUILD_SCOPE}",
       "CACHE_SAVE_SCOPE=${env.IMAGE_CACHE_MERGE_SCOPE}",
@@ -144,6 +145,7 @@ def premergeCacheImage() {
         ./build/new-jenkins/docker-with-flakey-network-protection.sh push $WEBPACK_BUILDER_PREFIX || true
         ./build/new-jenkins/docker-with-flakey-network-protection.sh push $YARN_RUNNER_PREFIX || true
         ./build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_RUNNER_PREFIX || true
+        ./build/new-jenkins/docker-with-flakey-network-protection.sh push $BASE_RUNNER_PREFIX || true
         ./build/new-jenkins/docker-with-flakey-network-protection.sh push $WEBPACK_CACHE_PREFIX
       """, label: 'upload cache images')
     }
@@ -156,6 +158,7 @@ def patchsetImage() {
 
     slackSendCacheBuild {
       withEnv([
+        "BASE_RUNNER_PREFIX=${env.BASE_RUNNER_PREFIX}",
         "CACHE_LOAD_SCOPE=${env.IMAGE_CACHE_MERGE_SCOPE}",
         "CACHE_LOAD_FALLBACK_SCOPE=${env.IMAGE_CACHE_BUILD_SCOPE}",
         "CACHE_SAVE_SCOPE=${cacheScope}",
@@ -189,6 +192,7 @@ def patchsetImage() {
       ./build/new-jenkins/docker-with-flakey-network-protection.sh push $WEBPACK_BUILDER_PREFIX || true
       ./build/new-jenkins/docker-with-flakey-network-protection.sh push $YARN_RUNNER_PREFIX || true
       ./build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_RUNNER_PREFIX || true
+      ./build/new-jenkins/docker-with-flakey-network-protection.sh push $BASE_RUNNER_PREFIX || true
       ./build/new-jenkins/docker-with-flakey-network-protection.sh push $WEBPACK_CACHE_PREFIX
     """, label: 'upload cache images')
   }