ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move file change detection into own stage 

refs DE-617

In order to properly split the stages for nospot removal, none of conditional jobs in Parallel Run Tests should directly depend on a heavyweight executor to know if they need to run.

Test Plan:
1. CDC Schema Check triggers when migrations change.
2. FSC triggers when specs change.
3. Local Docker Dev triggers when related files change.

[canvas-builds-refspec=3828a4bcd9f172fb6cec7046742cb678e3e489bf]

Change-Id: I578e4665d382aa27f03932d172604b20a8a1497d
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/262551
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Andrea Cirulli <andrea.cirulli@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>
This commit is contained in:
Aaron Ogata 2021-04-08 16:37:18 -07:00
parent f8fada9ef0
commit ddc875d672
2 changed files with 44 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
Jenkinsfile vendored
View File

@ -17,6 +17,7 @@
* You should have received a copy of the GNU Affero General Public License along * You should have received a copy of the GNU Affero General Public License along
* with this program. If not, see <http://www.gnu.org/licenses/>. * with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
def FILES_CHANGED_STAGE = "Detect Files Changed"
def JS_BUILD_IMAGE_STAGE = "Javascript (Build Image)" def JS_BUILD_IMAGE_STAGE = "Javascript (Build Image)"
def buildParameters = [ def buildParameters = [
@ -358,6 +359,25 @@ pipeline {
.timeout(2) .timeout(2)
.execute({ setupStage() }) .execute({ setupStage() })
extendedStage(FILES_CHANGED_STAGE)
.obeysAllowStages(false)
.handler(buildSummaryReport)
.timeout(2)
.execute { stageConfig ->
stageConfig.value('dockerDevFiles', git.changedFiles(dockerDevFiles, 'HEAD^'))
stageConfig.value('migrationFiles', sh(script: 'build/new-jenkins/check-for-migrations.sh', returnStatus: true) == 0)
dir(env.LOCAL_WORKDIR) {
stageConfig.value('specFiles', sh(script: '${WORKSPACE}/build/new-jenkins/spec-changes.sh', returnStatus: true) == 0)
}
// Remove the @tmp directory created by dir() for plugin builds, so bundler doesn't get confused.
// https://issues.jenkins.io/browse/JENKINS-52750
if(env.GERRIT_PROJECT != "canvas-lms") {
sh "rm -vrf $LOCAL_WORKDIR@tmp"
}
}
extendedStage('Rebase') extendedStage('Rebase')
.obeysAllowStages(false) .obeysAllowStages(false)
.handler(buildSummaryReport) .handler(buildSummaryReport)
@ -384,7 +404,7 @@ pipeline {
.timeout(10) .timeout(10)
.execute({ runMigrationsStage() }) .execute({ runMigrationsStage() })
stage('Parallel Run Tests') { extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { _, buildConfig ->
withEnv([ withEnv([
"CASSANDRA_IMAGE_TAG=${env.CASSANDRA_IMAGE}", "CASSANDRA_IMAGE_TAG=${env.CASSANDRA_IMAGE}",
"DYNAMODB_IMAGE_TAG=${env.DYNAMODB_IMAGE}", "DYNAMODB_IMAGE_TAG=${env.DYNAMODB_IMAGE}",
@ -425,47 +445,29 @@ pipeline {
string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"), string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
]) ])
if (sh(script: 'build/new-jenkins/check-for-migrations.sh', returnStatus: true) == 0) { echo 'adding CDC Schema check'
echo 'adding CDC Schema check' extendedStage('CDC Schema Check')
extendedStage('CDC Schema Check') .handler(buildSummaryReport)
.handler(buildSummaryReport) .required(buildConfig[FILES_CHANGED_STAGE].value('migrationFiles'))
.queue(stages, jobName: '/Canvas/cdc-event-transformer-master', buildParameters: buildParameters + [ .queue(stages, jobName: '/Canvas/cdc-event-transformer-master', buildParameters: buildParameters + [
string(name: 'CANVAS_LMS_IMAGE_PATH', value: "${env.PATCHSET_TAG}"), string(name: 'CANVAS_LMS_IMAGE_PATH', value: "${env.PATCHSET_TAG}"),
]) ])
}
else {
echo 'no migrations added, skipping CDC Schema check'
}
if ( echo 'adding Flakey Spec Catcher'
!configuration.isChangeMerged() && extendedStage('Flakey Spec Catcher')
( .handler(buildSummaryReport)
dir(env.LOCAL_WORKDIR){ (sh(script: '${WORKSPACE}/build/new-jenkins/spec-changes.sh', returnStatus: true) == 0) } || .required(!configuration.isChangeMerged() && buildConfig[FILES_CHANGED_STAGE].value('specFiles') || configuration.forceFailureFSC() == '1')
configuration.forceFailureFSC() == '1' .queue(stages, jobName: '/Canvas/test-suites/flakey-spec-catcher', buildParameters: buildParameters + [
) string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
) { string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
echo 'adding Flakey Spec Catcher' string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
extendedStage('Flakey Spec Catcher') ])
.handler(buildSummaryReport)
.queue(stages, jobName: '/Canvas/test-suites/flakey-spec-catcher', buildParameters: buildParameters + [
string(name: 'CASSANDRA_IMAGE_TAG', value: "${env.CASSANDRA_IMAGE_TAG}"),
string(name: 'DYNAMODB_IMAGE_TAG', value: "${env.DYNAMODB_IMAGE_TAG}"),
string(name: 'POSTGRES_IMAGE_TAG', value: "${env.POSTGRES_IMAGE_TAG}"),
])
}
// Flakey spec catcher using the dir step above will create this @tmp file, we need to remove it echo 'adding Local Docker Dev Build'
// https://issues.jenkins.io/browse/JENKINS-52750 extendedStage('Local Docker Dev Build')
if(!configuration.isChangeMerged() && env.GERRIT_PROJECT != "canvas-lms") { .handler(buildSummaryReport)
sh "rm -vrf $LOCAL_WORKDIR@tmp" .required(env.GERRIT_PROJECT == 'canvas-lms' && buildConfig[FILES_CHANGED_STAGE].value('dockerDevFiles'))
} .queue(stages, jobName: '/Canvas/test-suites/local-docker-dev-smoke', buildParameters: buildParameters)
if(env.GERRIT_PROJECT == 'canvas-lms' && git.changedFiles(dockerDevFiles, 'HEAD^')) {
echo 'adding Local Docker Dev Build'
extendedStage('Local Docker Dev Build')
.handler(buildSummaryReport)
.queue(stages, jobName: '/Canvas/test-suites/local-docker-dev-smoke', buildParameters: buildParameters)
}
extendedStage('Dependency Check') extendedStage('Dependency Check')
.handler(buildSummaryReport) .handler(buildSummaryReport)

4
build/new-jenkins/check-for-migrations.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# grep will exit with code 0 if any commits between origin/master and # grep will exit with code 0 if any commits between HEAD^ and
# HEAD contain db/migrate. # HEAD contain db/migrate.
git show --pretty="" --name-only origin/master..HEAD | grep "db/migrate" git show --pretty="" --name-only HEAD^..HEAD | grep "db/migrate"