separate ruby image into gems / final

refs DE-164

Test Plan:
1. Ensure that the pre-merge, no-cache path works.
2. Ensure that the post-merge, no-cache path works.
3. Ensure that the cache path works.
4. Ensure that local docker setup scripts still work.

Change-Id: I1ecb6d7c37346b29816a2202e27ef1e662a3ac60
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/244290
Reviewed-by: James Butters <jbutters@instructure.com>
QA-Review: James Butters <jbutters@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>

Dockerfile

@@ -31,7 +31,6 @@ RUN set -eux; \
 USER docker
 COPY --chown=docker:docker babel.config.js    ${APP_HOME}
 COPY --chown=docker:docker build/new-jenkins  ${APP_HOME}build/new-jenkins
-COPY --chown=docker:docker gems               ${APP_HOME}gems
 COPY --chown=docker:docker package.json       ${APP_HOME}
 COPY --chown=docker:docker packages           ${APP_HOME}packages
 COPY --chown=docker:docker script             ${APP_HOME}script

Jenkinsfile

@@ -160,6 +160,10 @@ pipeline {
     RUBY_MERGE_IMAGE = "$RUBY_IMAGE:$GERRIT_BRANCH"
     RUBY_PATCHSET_IMAGE = "$RUBY_IMAGE:$NAME-$TAG_SUFFIX"
 
+    RUBY_GEMS_IMAGE = "$BUILD_IMAGE-ruby-gems-only"
+    RUBY_GEMS_MERGE_IMAGE = "$RUBY_GEMS_IMAGE:$GERRIT_BRANCH"
+    RUBY_GEMS_PATCHSET_IMAGE = "$RUBY_GEMS_IMAGE:$NAME-$TAG_SUFFIX"
+
     CASSANDRA_IMAGE_TAG=imageTag.cassandra()
     DYNAMODB_IMAGE_TAG=imageTag.dynamodb()
     POSTGRES_IMAGE_TAG=imageTag.postgres()
@@ -261,6 +265,7 @@ pipeline {
                     sh 'docker tag $MERGE_TAG $PATCHSET_TAG'
                   } else {
                     sh 'build/new-jenkins/docker-build.sh'
+                    sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_GEMS_PATCHSET_IMAGE"
                     sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_PATCHSET_IMAGE"
                   }
                   sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $PATCHSET_TAG"
@@ -383,6 +388,10 @@ pipeline {
                   // Retriggers won't have an image to tag/push, pull that
                   // image if doesn't exist. If image is not found it will
                   // return NULL
+                  if (!sh (script: 'docker images -q $RUBY_GEMS_PATCHSET_IMAGE')) {
+                    sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $RUBY_GEMS_PATCHSET_IMAGE'
+                  }
+
                   if (!sh (script: 'docker images -q $RUBY_PATCHSET_IMAGE')) {
                     sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $RUBY_PATCHSET_IMAGE'
                   }
@@ -393,9 +402,11 @@ pipeline {
 
                   // publish canvas-lms:$GERRIT_BRANCH (i.e. canvas-lms:master)
                   sh 'docker tag $PUBLISHABLE_TAG $MERGE_TAG'
+                  sh 'docker tag $RUBY_GEMS_PATCHSET_IMAGE $RUBY_GEMS_MERGE_IMAGE'
                   sh 'docker tag $RUBY_PATCHSET_IMAGE $RUBY_MERGE_IMAGE'
                   // push *all* canvas-lms images (i.e. all canvas-lms prefixed tags)
                   sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $MERGE_TAG'
+                  sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_GEMS_MERGE_IMAGE'
                   sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $RUBY_MERGE_IMAGE'
                 }
               }

build/new-jenkins/docker-build.sh

@@ -9,7 +9,20 @@ optionalFromCache=''
 [[ "${SKIP_CACHE:-false}" = "false" ]] && optionalFromCache="--cache-from $MERGE_TAG"
 
 optionalFromCacheRuby=''
-[[ "${SKIP_CACHE:-false}" = "false" ]] && optionalFromCacheRuby="--cache-from $RUBY_MERGE_IMAGE"
+[[ "${SKIP_CACHE:-false}" = "false" ]] && optionalFromCacheRuby="--cache-from $RUBY_GEMS_MERGE_IMAGE --cache-from $RUBY_GEMS_PATCHSET_IMAGE"
+
+# shellcheck disable=SC2086
+DOCKER_BUILDKIT=1 docker build \
+  --pull \
+  --build-arg ALPINE_MIRROR="$ALPINE_MIRROR" \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  --build-arg POSTGRES_CLIENT="$POSTGRES_CLIENT" \
+  --build-arg RUBY="$RUBY" \
+  --file ruby.Dockerfile \
+  $optionalFromCacheRuby \
+  --tag "$RUBY_GEMS_PATCHSET_IMAGE" \
+  --target gems-only \
+  "$WORKSPACE"
 
 # shellcheck disable=SC2086
 DOCKER_BUILDKIT=1 docker build \
@@ -21,6 +34,7 @@ DOCKER_BUILDKIT=1 docker build \
   --file ruby.Dockerfile \
   $optionalFromCacheRuby \
   --tag "$RUBY_PATCHSET_IMAGE" \
+  --target final \
   "$WORKSPACE"
 
 # shellcheck disable=SC2086
@@ -28,7 +42,7 @@ DOCKER_BUILDKIT=1 docker build \
   --build-arg ALPINE_MIRROR="$ALPINE_MIRROR" \
   --build-arg BUILDKIT_INLINE_CACHE=1 \
   --build-arg NODE="$NODE" \
-  --build-arg RUBY_PATCHSET_IMAGE="$RUBY_PATCHSET_IMAGE" \
+  --build-arg RUBY_PATCHSET_IMAGE="$RUBY_GEMS_PATCHSET_IMAGE" \
   --file Dockerfile \
   $optionalFromCache \
   --tag "$PATCHSET_TAG" \

ruby.Dockerfile

@@ -1,6 +1,6 @@
 ARG RUBY=2.6
 
-FROM instructure/passenger-nginx-alpine:${RUBY}
+FROM instructure/passenger-nginx-alpine:${RUBY} AS gems-only
 LABEL maintainer="Instructure"
 
 ARG POSTGRES_CLIENT=12.2
@@ -92,4 +92,5 @@ RUN set -eux; \
   && bundle install --jobs $(nproc) \
   && rm -rf $GEM_HOME/cache
 
+FROM gems-only AS final
 COPY --chown=docker:docker . $APP_HOME