optimize webpack linters build time
refs DE-938 flag=none TEST PLAN: Confirm all builds still run successfully Confirm all webpack linter jobs still run Comfirm bundle sizes are appropriately reported This task removes the webpack linter stage from premerge builds, spreads the previous tasks around, and adds a step to postmerge builds to track and report webpack bundle sizes Change-Id: I017876726ac0db6f099367b1ecfd086cfb726cf8 Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/289680 Reviewed-by: Aaron Ogata <aogata@instructure.com> Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com> QA-Review: Bobby Buten <bobby.buten@instructure.com> Product-Review: Bobby Buten <bobby.buten@instructure.com>
This commit is contained in:
Bobby Buten
parent
4e84cec463
commit
1b3a8216ae
|
|
@ -306,6 +306,7 @@ pipeline {
|
|||
DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_UNIQUE_SCOPE"
|
||||
POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_UNIQUE_SCOPE"
|
||||
WEBPACK_BUILDER_IMAGE = "$WEBPACK_BUILDER_PREFIX:$IMAGE_CACHE_UNIQUE_SCOPE"
|
||||
WEBPACK_CACHE_IMAGE = "$WEBPACK_CACHE_PREFIX:$IMAGE_CACHE_UNIQUE_SCOPE"
|
||||
|
||||
CASSANDRA_MERGE_IMAGE = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-${env.RSPEC_PROCESSES ?: '4'}"
|
||||
DYNAMODB_MERGE_IMAGE = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-${env.RSPEC_PROCESSES ?: '4'}"
|
||||
|
|
@ -498,6 +499,19 @@ pipeline {
|
|||
gerrit.submitLintReview('-2', 'This commit contains only changes to config/locales/, this could be a bad sign!')
|
||||
}
|
||||
|
||||
extendedStage('Webpack ES-Check')
|
||||
.hooks(buildSummaryReportHooks.call())
|
||||
.obeysAllowStages(false)
|
||||
.required(env.GERRIT_CHANGE_ID != '0')
|
||||
.execute { webpackStage.&runESCheck() }
|
||||
|
||||
extendedStage('Webpack Bundle Size Check')
|
||||
.hooks(buildSummaryReportHooks.call())
|
||||
.obeysAllowStages(false)
|
||||
.required(configuration.isChangeMerged())
|
||||
.timeout(20)
|
||||
.execute { webpackStage.&calcBundleSizes() }
|
||||
|
||||
extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
|
||||
def stages = [:]
|
||||
|
||||
|
|
@ -562,7 +576,6 @@ pipeline {
|
|||
callableWithDelegate(lintersStage.featureFlagStage(nestedStages, buildConfig))()
|
||||
callableWithDelegate(lintersStage.groovyStage(nestedStages, buildConfig))()
|
||||
callableWithDelegate(lintersStage.masterBouncerStage(nestedStages))()
|
||||
callableWithDelegate(lintersStage.webpackStage(nestedStages))()
|
||||
callableWithDelegate(lintersStage.yarnStage(nestedStages, buildConfig))()
|
||||
|
||||
parallel(nestedStages)
|
||||
|
|
|
|||
|
|
@ -1,54 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
#
|
||||
# Copyright (C) 2016 - present Instructure, Inc.
|
||||
#
|
||||
# This file is part of Canvas.
|
||||
#
|
||||
# Canvas is free software: you can redistribute it and/or modify it under
|
||||
# the terms of the GNU Affero General Public License as published by the Free
|
||||
# Software Foundation, version 3 of the License.
|
||||
#
|
||||
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License along
|
||||
# with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# gergich capture custom:./gergich/compile_assets:Gergich::CompileAssets \
|
||||
# "bundle exec rake RAILS_ENV=test canvas:compile_assets[$GENERATE_DOCS,$CHECK_SYNTAX,$COMPILE_STYLEGUIDE,$BUILD_JS]"
|
||||
class Gergich::CompileAssets
|
||||
def run(output)
|
||||
# HBS
|
||||
pattern = %r| # Example:
|
||||
^HBS\sPRECOMPILATION\sFAILED\n # HBS PRECOMPILATION FAILED
|
||||
([^:\n]+):(\d+):\s([^\n]+\n # app/views/jst/googleDocsTreeView.handlebars:3 Parse error:
|
||||
(\s\s[^\n]+\n)* # ...le="menuitem"> {{t} {{name}} <ul>
|
||||
# ----------------------^
|
||||
# Expecting 'CLOSE', 'CLOSE_UNESCAPED', 'STRING', ...
|
||||
)
|
||||
|mx
|
||||
|
||||
result = output.scan(pattern).map do |file, line, error|
|
||||
error.sub!(/\n/, "\n\n") # separate first line from the rest, which will be indented (monospace)
|
||||
{ path: file, message: error, position: line.to_i, severity: "error" }
|
||||
end
|
||||
|
||||
# COFFEE
|
||||
cwd = Dir.pwd
|
||||
puts cwd
|
||||
|
||||
pattern = %r{ # Example:
|
||||
^#{cwd}/([^\n]+?):(\d+):\d+:\serror:\s([^\n]+)\n # /absolute/path/to/file.coffee:7:1: error: unexpected INDENT
|
||||
([^\n]+)\n # falseList = []
|
||||
([^\n]+)\n # ^^^^^
|
||||
}mx
|
||||
|
||||
result.concat(output.scan(pattern).map do |file, line, error, context1, context2|
|
||||
error = "#{error}\n\n #{context1}\n #{context2}"
|
||||
{ path: file, message: error, position: line.to_i, severity: "error" }
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
@ -211,9 +211,10 @@ else
|
|||
[ "$WEBPACK_BUILDER_SELECTED_TAG" != "${WEBPACK_BUILDER_TAGS[SAVE_TAG]}" ] && tag_remote_async "WEBPACK_BUILDER_TAG_REMOTE_SAVE_PID" $WEBPACK_BUILDER_SELECTED_TAG ${WEBPACK_BUILDER_TAGS[SAVE_TAG]}
|
||||
|
||||
[ ! -z "${CACHE_UNIQUE_SCOPE-}" ] && tag_remote_async "WEBPACK_BUILDER_TAG_REMOTE_UNIQUE_PID" $WEBPACK_BUILDER_SELECTED_TAG ${WEBPACK_BUILDER_TAGS[UNIQUE_TAG]}
|
||||
[ ! -z "${CACHE_UNIQUE_SCOPE-}" ] && tag_remote_async "WEBPACK_CACHE_TAG_REMOTE_UNIQUE_PID" $WEBPACK_CACHE_SELECTED_TAG ${WEBPACK_CACHE_TAGS[UNIQUE_TAG]}
|
||||
fi
|
||||
|
||||
tag_many $WEBPACK_CACHE_SELECTED_TAG local/webpack-cache ${WEBPACK_CACHE_TAGS[SAVE_TAG]}
|
||||
tag_many $WEBPACK_CACHE_SELECTED_TAG local/webpack-cache ${WEBPACK_CACHE_TAGS[SAVE_TAG]} ${WEBPACK_CACHE_TAGS[UNIQUE_TAG]-}
|
||||
|
||||
# Build Final Image
|
||||
if [ -n "${1:-}" ]; then
|
||||
|
|
|
|||
|
|
@ -78,15 +78,6 @@ def masterBouncerStage(stages) {
|
|||
}
|
||||
}
|
||||
|
||||
def webpackStage(stages) {
|
||||
{ ->
|
||||
callableWithDelegate(queueTestStage())(stages,
|
||||
name: 'webpack',
|
||||
command: './build/new-jenkins/linters/run-gergich-webpack.sh'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def featureFlagStage(stages, buildConfig) {
|
||||
{ ->
|
||||
extendedStage('Linters - feature-flag')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
/*
|
||||
* Copyright (C) 2022 - present Instructure, Inc.
|
||||
*
|
||||
* This file is part of Canvas.
|
||||
*
|
||||
* Canvas is free software: you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Affero General Public License as published by the Free
|
||||
* Software Foundation, version 3 of the License.
|
||||
*
|
||||
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License along
|
||||
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
def calcBundleSizes() {
|
||||
sh '''
|
||||
docker run --name webpack-linters \
|
||||
$PATCHSET_TAG bash -c "./build/new-jenkins/record-webpack-sizes.sh"
|
||||
'''
|
||||
|
||||
sh 'docker cp webpack-linters:/tmp/big_bundles.csv tmp/big_bundles.csv'
|
||||
|
||||
def bigFileList = []
|
||||
def thingsWeKnowAreWayTooBig = [
|
||||
'account_settings',
|
||||
'assignment_edit',
|
||||
'assignment_index',
|
||||
'calendar',
|
||||
'discussion_topic_edit',
|
||||
'discussion_topics_post',
|
||||
'edit_rubric',
|
||||
'manage_groups',
|
||||
'wiki_page_show'
|
||||
]
|
||||
|
||||
def csv = readCSV file: 'tmp/big_bundles.csv'
|
||||
for (def records : csv) {
|
||||
|
||||
// skip auto-generated bundles
|
||||
if (records[0] =~ /^\d/) {
|
||||
continue
|
||||
}
|
||||
|
||||
reportToSplunk('webpack_bundle_size', [
|
||||
'fileName': records[0],
|
||||
'fileSize': records[1],
|
||||
'version': '3a',
|
||||
])
|
||||
|
||||
int fileSize = records[1].toInteger()
|
||||
// should match maxAssetSize value in ~/ui-build/webpack/index.js
|
||||
if ((fileSize > 1400000) && !thingsWeKnowAreWayTooBig.any{records[0].contains(it)}) {
|
||||
bigFileList.push(records[0])
|
||||
}
|
||||
}
|
||||
|
||||
if (!bigFileList.isEmpty()) {
|
||||
def authorSlackId = env.GERRIT_EVENT_ACCOUNT_EMAIL ? slackUserIdFromEmail(email: env.GERRIT_EVENT_ACCOUNT_EMAIL, botUser: true, tokenCredentialId: 'slack-user-id-lookup') : ''
|
||||
def authorSlackMsg = authorSlackId ? "<@$authorSlackId>" : env.GERRIT_EVENT_ACCOUNT_NAME
|
||||
def authorSegment = "Patchset <${env.GERRIT_CHANGE_URL}|#${env.GERRIT_CHANGE_NUMBER}> by ${authorSlackMsg} includes an asset bundle over the recommended max size"
|
||||
def bigFiles = bigFileList.join(", ")
|
||||
def extra = "Bundle ${bigFiles} is too big and not in our known list of oversized bundles. This patchset may have pushed in over the acceptable limit. Please review the <https://inst.splunkcloud.com/en-US/app/search/report?sid=scheduler_cm9iZXJ0LmJ1dGVuQGluc3RydWN0dXJlLmNvbQ__search__RMD5ca61b6204ef60fe9_at_1650416400_45040_1987C8A1-0299-4A02-A75B-503AB27123E0&s=%2FservicesNS%2Fnobody%2Fsearch%2Fsaved%2Fsearches%2Fwebpack_bundle_size|Webpack Bundle Size> to see how bundles have grown over time."
|
||||
def summaryUrl = "${env.BUILD_URL}/build-summary-report"
|
||||
|
||||
slackSend(
|
||||
channel: '#canvas_builds',
|
||||
color: 'warning',
|
||||
message: "${authorSegment}. Build <${summaryUrl}|#${env.BUILD_NUMBER}>\n\n$extra"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def runESCheck() {
|
||||
sh './build/new-jenkins/docker-with-flakey-network-protection.sh pull $WEBPACK_CACHE_IMAGE'
|
||||
|
||||
def statusCode = sh(script: '''
|
||||
docker run --name webpack-cache \
|
||||
$WEBPACK_CACHE_IMAGE bash -c \
|
||||
"yarn add es-check@5 --silent && node_modules/.bin/es-check es10 ./public/dist/**/*.js > /tmp/escheck.out"
|
||||
''', returnStatus:true)
|
||||
|
||||
if (statusCode != 0) {
|
||||
sh 'docker cp webpack-cache:/tmp/escheck.out tmp/escheck.out'
|
||||
|
||||
def data = readFile(file: 'tmp/escheck.out')
|
||||
echo data
|
||||
gerrit.submitLintReview('-2', 'ES version matching errors exist')
|
||||
}
|
||||
}
|
||||
|
|
@ -32,5 +32,7 @@ ruby script/rlint --no-fail-on-offense
|
|||
ruby script/lint_commit_message
|
||||
node script/yarn-validate-workspace-deps.js 2>/dev/null < <(yarn --silent workspaces info --json)
|
||||
|
||||
rake css:styleguide doc:api
|
||||
|
||||
gergich status
|
||||
echo "LINTER OK!"
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
export COMPILE_ASSETS_NPM_INSTALL=0
|
||||
export COMPILE_ASSETS_BRAND_CONFIGS=0
|
||||
export JS_BUILD_NO_FALLBACK=1
|
||||
gergich capture custom:./build/gergich/compile_assets:Gergich::CompileAssets 'rake canvas:compile_assets'
|
||||
yarn lint:browser-code
|
||||
gergich status
|
||||
echo "WEBPACK_BUILD OK!"
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
set -o errexit -o errtrace -o nounset -o pipefail
|
||||
|
||||
DANGER_FILE_SIZE=1000000
|
||||
|
||||
cd /usr/src/app/public/dist/webpack-production
|
||||
ls -ltra
|
||||
for file in *.js; do
|
||||
filesize=$(ls -l $file | awk '{print $5}')
|
||||
if [ $filesize -gt $DANGER_FILE_SIZE ]; then
|
||||
echo "$file,$filesize" >> /tmp/big_bundles.csv
|
||||
fi
|
||||
done
|
||||
|
|
@ -68,20 +68,8 @@ module.exports = {
|
|||
// this number to the size of our biggest known asset and hopefully someday get
|
||||
// to where they are all under the default value of 250000 and then remove this
|
||||
// TODO: decrease back to 1200000 LS-1222
|
||||
maxAssetSize: 1400000,
|
||||
assetFilter: assetFilename => {
|
||||
const thingsWeKnowAreWayTooBig = [
|
||||
'assignment_edit',
|
||||
'canvas-rce-async-chunk',
|
||||
'canvas-rce-old-async-chunk',
|
||||
'discussion_topic_edit',
|
||||
'discussion_topics_post'
|
||||
]
|
||||
return (
|
||||
assetFilename.endsWith('.js') &&
|
||||
!thingsWeKnowAreWayTooBig.some(t => assetFilename.includes(t))
|
||||
)
|
||||
}
|
||||
// NOTE: if maxAssetSize changes, update: ~build/new-jenkins/library/vars/webpackStage.groovy
|
||||
maxAssetSize: 1400000
|
||||
},
|
||||
optimization: {
|
||||
// concatenateModules: false, // uncomment if you want to get more accurate stuff from `yarn webpack:analyze`
|
||||
|
|
|
|||
Loading…
Reference in New Issue