ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

reduce file dependencies for webpack build 

refs DE-212

If a user isn’t changing any front-end related files, we can get a significant time savings by not re-building webpack. In order to do this, we have to remove the multi-layer dependencies image due to poor docker support for multi-layer caching. The dependencies build is an artifact of the ruby-only image that no longer exists, so no behavioural changes are expected.

Test Plan:
1. Ensure that the post-merge / pre-merge caches are used on post-merge build.
2. Ensure that the pre-merge cache is used on a pre-merge build.
3. Ensure that both above work on specifically the same / different nodes.

[change-merged]
[build-registry-path=jenkins/canvas-lms/de-212]
[skip-rebase]

Change-Id: I659db8f201bfe5e4fa6d20d802a639f5a55685df
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/245976
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: James Butters <jbutters@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>
This commit is contained in:
Aaron Ogata 2020-08-25 07:51:49 -07:00
parent 9e5c040bb0
commit 7beea9fcf6
5 changed files with 97 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Dockerfile
View File

@ -4,7 +4,7 @@
ARG RUBY=2.6-p6.0.4
FROM instructure/ruby-passenger:$RUBY AS dependencies
FROM instructure/ruby-passenger:$RUBY AS webpack-final
LABEL maintainer="Instructure"
ARG POSTGRES_CLIENT=12
@ -118,8 +118,7 @@ COPY --chown=docker:docker script ${APP_HOME}script
RUN yarn postinstall
FROM dependencies AS webpack-final
ARG JS_BUILD_NO_UGLIFY=0
COPY --chown=docker:docker . ${APP_HOME}
ARG JS_BUILD_NO_UGLIFY=0
RUN COMPILE_ASSETS_NPM_INSTALL=0 JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" bundle exec rails canvas:compile_assets

36
Dockerfile.jenkins
View File

@ -23,7 +23,15 @@ RUN --mount=target=/tmp/src find gems/plugins/* -name 'package.json' -exec cp --
RUN --mount=target=/tmp/src find packages/* -name 'package.json' -exec cp --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find client_apps/* -name 'package.json' -exec cp --parents {} /tmp/dst \;
FROM instructure/ruby-passenger:$RUBY AS dependencies
FROM busybox AS cache-helper-collect-webpack
RUN mkdir -p /tmp/dst /tmp/dst
WORKDIR /tmp/src
RUN --mount=target=/tmp/src find gems -type d -not -path "gems/plugins/*" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/coffeescripts" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/jsx" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/views/jst" -exec cp -rf --parents {} /tmp/dst \;
FROM instructure/ruby-passenger:$RUBY AS webpack-final
LABEL maintainer="Instructure"
ARG POSTGRES_CLIENT=12
@ -137,8 +145,30 @@ COPY --chown=docker:docker script ${APP_HOME}script
RUN yarn postinstall
FROM dependencies AS webpack-final
COPY --chown=docker:docker app/coffeescripts ${APP_HOME}app/coffeescripts
COPY --chown=docker:docker app/jsx ${APP_HOME}app/jsx
COPY --chown=docker:docker app/stylesheets ${APP_HOME}app/stylesheets
COPY --chown=docker:docker app/views/jst ${APP_HOME}app/views/jst
COPY --chown=docker:docker bin ${APP_HOME}bin
COPY --chown=docker:docker client_apps ${APP_HOME}client_apps
COPY --chown=docker:docker config ${APP_HOME}config
COPY --chown=docker:docker db/migrate/*_regenerate_brand_files_based_on_new_defaults_*.rb ${APP_HOME}db/migrate/
COPY --chown=docker:docker frontend_build ${APP_HOME}frontend_build
COPY --chown=docker:docker --from=cache-helper-collect-webpack /tmp/dst ${APP_HOME}
COPY --chown=docker:docker lib ${APP_HOME}lib
COPY --chown=docker:docker public ${APP_HOME}public
COPY --chown=docker:docker Rakefile ${APP_HOME}
COPY --chown=docker:docker gulpfile.js ${APP_HOME}
COPY --chown=docker:docker webpack.config.js ${APP_HOME}
COPY --chown=docker:docker .bowerrc ${APP_HOME}
COPY --chown=docker:docker .i18nignore ${APP_HOME}
COPY --chown=docker:docker .i18nrc ${APP_HOME}
ARG JS_BUILD_NO_UGLIFY=0
RUN COMPILE_ASSETS_API_DOCS=0 COMPILE_ASSETS_NPM_INSTALL=0 COMPILE_ASSETS_STYLEGUIDE=0 JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" bundle exec rails canvas:compile_assets
COPY --chown=docker:docker app ${APP_HOME}/app
COPY --chown=docker:docker doc ${APP_HOME}/doc
RUN bundle exec rails doc:api css:styleguide
COPY --chown=docker:docker . ${APP_HOME}
RUN COMPILE_ASSETS_NPM_INSTALL=0 JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" bundle exec rails canvas:compile_assets

32
Jenkinsfile vendored
View File

@ -272,10 +272,7 @@ pipeline {
NODE = configuration.node()
RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
DEPENDENCIES_IMAGE = getDependenciesImage()
DEPENDENCIES_MERGE_IMAGE = getDependenciesMergeImage()
DEPENDENCIES_PATCHSET_IMAGE = getDependenciesPatchsetImage()
CACHE_IMAGE = "$BUILD_IMAGE-cache:$GERRIT_BRANCH"
CASSANDRA_IMAGE_TAG=imageTag.cassandra()
DYNAMODB_IMAGE_TAG=imageTag.dynamodb()
@ -425,11 +422,11 @@ pipeline {
sh 'docker tag $MERGE_TAG $PATCHSET_TAG'
} else {
withEnv([
"CACHE_TAG=${configuration.isChangeMerged() ? env.MERGE_TAG : env.CACHE_IMAGE}",
"JS_BUILD_NO_UGLIFY=${configuration.isChangeMerged() ? 0 : 1}"
]) {
sh 'build/new-jenkins/docker-build.sh'
sh "build/new-jenkins/docker-build.sh $PATCHSET_TAG"
}
sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $DEPENDENCIES_PATCHSET_IMAGE"
}
sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push $PATCHSET_TAG"
if (isPatchsetPublishable()) {
@ -461,6 +458,22 @@ pipeline {
"POSTGRES_IMAGE_TAG=${migrations.postgresTag()}"
]) {
def stages = [:]
if (configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms') {
echo 'adding Build Docker Image Cache'
stages['Build Docker Image Cache'] = {
skipIfPreviouslySuccessful("build-docker-cache") {
withEnv([
"CACHE_TAG=${env.CACHE_IMAGE}",
"JS_BUILD_NO_UGLIFY=1"
]) {
sh "build/new-jenkins/docker-build.sh $CACHE_IMAGE"
sh "build/new-jenkins/docker-with-flakey-network-protection.sh push $CACHE_IMAGE"
}
}
}
}
if (!configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms') {
echo 'adding Linters'
timedStage('Linters', stages, {
@ -582,25 +595,18 @@ pipeline {
// Retriggers won't have an image to tag/push, pull that
// image if doesn't exist. If image is not found it will
// return NULL
if (!sh (script: 'docker images -q $DEPENDENCIES_PATCHSET_IMAGE')) {
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $DEPENDENCIES_PATCHSET_IMAGE'
}
if (!sh (script: 'docker images -q $PATCHSET_TAG')) {
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $PATCHSET_TAG'
}
// publish canvas-lms:$GERRIT_BRANCH (i.e. canvas-lms:master)
sh 'docker tag $PUBLISHABLE_TAG $MERGE_TAG'
sh 'docker tag $DEPENDENCIES_PATCHSET_IMAGE $DEPENDENCIES_MERGE_IMAGE'
def GIT_REV = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
sh "docker tag \$PUBLISHABLE_TAG \$BUILD_IMAGE:${GIT_REV}"
sh "docker tag \$DEPENDENCIES_PATCHSET_IMAGE \$DEPENDENCIES_IMAGE:${GIT_REV}"
// push *all* canvas-lms images (i.e. all canvas-lms prefixed tags)
sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $BUILD_IMAGE'
sh './build/new-jenkins/docker-with-flakey-network-protection.sh push $DEPENDENCIES_IMAGE'
}
}
}

43
build/Dockerfile.template
View File

@ -2,7 +2,7 @@
# syntax = docker/dockerfile:1.0-experimental
<% end -%>
<%= generation_message %>
ARG RUBY=2.6
ARG RUBY=2.6-p6.0.4
<% if jenkins? -%>
FROM busybox AS cache-helper-collect-gems
@ -20,8 +20,16 @@ RUN mkdir -p /tmp/dst /tmp/dst
WORKDIR /tmp/src
<%= yarn_packages.map { |p| "RUN --mount=target=/tmp/src find #{p} -name 'package.json' -exec cp --parents {} /tmp/dst \\;" }.join("\n") %>
FROM busybox AS cache-helper-collect-webpack
RUN mkdir -p /tmp/dst /tmp/dst
WORKDIR /tmp/src
RUN --mount=target=/tmp/src find gems -type d -not -path "gems/plugins/*" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/coffeescripts" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/jsx" -exec cp -rf --parents {} /tmp/dst \;
RUN --mount=target=/tmp/src find gems/plugins -path "app/views/jst" -exec cp -rf --parents {} /tmp/dst \;
<% end -%>
FROM instructure/ruby-passenger:$RUBY AS dependencies
FROM instructure/ruby-passenger:$RUBY AS webpack-final
LABEL maintainer="Instructure"
ARG POSTGRES_CLIENT=12
@ -110,8 +118,37 @@ COPY --chown=docker:docker script ${APP_HOME}script
RUN yarn postinstall
FROM dependencies AS webpack-final
<% if jenkins? -%>
COPY --chown=docker:docker app/coffeescripts ${APP_HOME}app/coffeescripts
COPY --chown=docker:docker app/jsx ${APP_HOME}app/jsx
COPY --chown=docker:docker app/stylesheets ${APP_HOME}app/stylesheets
COPY --chown=docker:docker app/views/jst ${APP_HOME}app/views/jst
COPY --chown=docker:docker bin ${APP_HOME}bin
COPY --chown=docker:docker client_apps ${APP_HOME}client_apps
COPY --chown=docker:docker config ${APP_HOME}config
COPY --chown=docker:docker db/migrate/*_regenerate_brand_files_based_on_new_defaults_*.rb ${APP_HOME}db/migrate/
COPY --chown=docker:docker frontend_build ${APP_HOME}frontend_build
COPY --chown=docker:docker --from=cache-helper-collect-webpack /tmp/dst ${APP_HOME}
COPY --chown=docker:docker lib ${APP_HOME}lib
COPY --chown=docker:docker public ${APP_HOME}public
COPY --chown=docker:docker Rakefile ${APP_HOME}
COPY --chown=docker:docker gulpfile.js ${APP_HOME}
COPY --chown=docker:docker webpack.config.js ${APP_HOME}
COPY --chown=docker:docker .bowerrc ${APP_HOME}
COPY --chown=docker:docker .i18nignore ${APP_HOME}
COPY --chown=docker:docker .i18nrc ${APP_HOME}
ARG JS_BUILD_NO_UGLIFY=0
RUN COMPILE_ASSETS_API_DOCS=0 COMPILE_ASSETS_NPM_INSTALL=0 COMPILE_ASSETS_STYLEGUIDE=0 JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" bundle exec rails canvas:compile_assets
COPY --chown=docker:docker app ${APP_HOME}/app
COPY --chown=docker:docker doc ${APP_HOME}/doc
RUN bundle exec rails doc:api css:styleguide
COPY --chown=docker:docker . ${APP_HOME}
<% else -%>
COPY --chown=docker:docker . ${APP_HOME}
ARG JS_BUILD_NO_UGLIFY=0
RUN COMPILE_ASSETS_NPM_INSTALL=0 JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" bundle exec rails canvas:compile_assets
<% end -%>

16
build/new-jenkins/docker-build.sh
View File

@ -3,8 +3,6 @@
set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
WORKSPACE=${WORKSPACE:-$(pwd)}
RUBY_PATCHSET_IMAGE=${RUBY_PATCHSET_IMAGE:-canvas-lms-ruby}
PATCHSET_TAG=${PATCHSET_TAG:-canvas-lms}
dependencyArgs=(
--build-arg BUILDKIT_INLINE_CACHE=1
@ -13,22 +11,12 @@ dependencyArgs=(
--file Dockerfile.jenkins
)
if [[ "${SKIP_CACHE:-false}" = "false" ]]; then
dependencyArgs+=("--cache-from $DEPENDENCIES_MERGE_IMAGE")
fi
# shellcheck disable=SC2086
DOCKER_BUILDKIT=1 PROGRESS_NO_TRUNC=1 docker build \
--pull \
${dependencyArgs[@]} \
--tag "$DEPENDENCIES_PATCHSET_IMAGE" \
--target dependencies \
"$WORKSPACE"
# shellcheck disable=SC2086
DOCKER_BUILDKIT=1 PROGRESS_NO_TRUNC=1 docker build \
${dependencyArgs[@]} \
--build-arg JS_BUILD_NO_UGLIFY="$JS_BUILD_NO_UGLIFY" \
--tag "$PATCHSET_TAG" \
--cache-from $CACHE_TAG \
--tag "$1" \
--target webpack-final \
"$WORKSPACE"