cache migrations by md5sum

refs DE-23

[change-merged]
[build-registry-path=jenkins/canvas-lms/de-23-test]

flag = none

Test Plan:
1. Jenkins pre-merge build with a new migration re-builds migrations
2. Jenkins pre-merge build with migrations cached does not re-build
3. Jenkins post-merge build with migrations cached re-builds migrations
4. skip-cache directive does not use the cached migration
5. Jenkins non-main builds works as expected
6. Jenkins FSC works as expected

Change-Id: I3221400a15220884740ad5136de7185a88ae2b39
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/241013
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Aaron Ogata <aogata@instructure.com>
Reviewed-by: Ryan Norton <rnorton@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>

Jenkinsfile

@@ -434,8 +434,28 @@ pipeline {
               }
             }
 
+            def migrations = load('build/new-jenkins/groovy/migrations.groovy')
+
+            stage ('Run Migrations') {
+              timeout(time: 10) {
+                skipIfPreviouslySuccessful('run-migrations') {
+                  withEnv([
+                    "COMPOSE_FILE=docker-compose.new-jenkins.yml",
+                    "POSTGRES_PASSWORD=sekret"
+                  ]) {
+                    migrations.runMigrations()
+                    sh 'docker-compose down --remove-orphans'
+                  }
+                }
+              }
+            }
+
             stage('Parallel Run Tests') {
-              withEnv([]) {
+              withEnv([
+                "CASSANDRA_IMAGE_TAG=${migrations.cassandraTag()}",
+                "DYNAMODB_IMAGE_TAG=${migrations.dynamodbTag()}",
+                "POSTGRES_IMAGE_TAG=${migrations.postgresTag()}"
+              ]) {
                 def stages = [:]
                 if (!configuration.isChangeMerged() && env.GERRIT_PROJECT == 'canvas-lms') {
                   echo 'adding Linters'

Jenkinsfile.selenium.flakey_spec_catcher

@@ -64,7 +64,6 @@ def setupNode() {
   sh 'build/new-jenkins/docker-compose-pull.sh'
   sh 'build/new-jenkins/docker-compose-pull-selenium.sh'
   sh 'build/new-jenkins/docker-compose-build-up.sh'
-  sh 'build/new-jenkins/docker-compose-setup-databases.sh'
 }
 
 def computeTestCount() {

Jenkinsfile.vendored-gems

@@ -70,7 +70,6 @@ pipeline {
                 checkout scm
                 sh 'build/new-jenkins/docker-compose-pull.sh'
                 sh 'build/new-jenkins/docker-compose-build-up.sh'
-                sh 'build/new-jenkins/docker-compose-setup-cassandra-keyspaces.sh'
               }
             }
 

build/new-jenkins/consumer-smoke-test.sh

@@ -8,6 +8,7 @@ export COMPOSE_FILE='docker-compose.new-jenkins.consumer.yml'
 ./build/new-jenkins/docker-compose-pull.sh
 
 docker-compose up -d
+docker-compose exec -T postgres /bin/bash -c /wait-for-it
 docker-compose run -T canvas bundle exec rails db:create db:migrate
 
 docker ps
@@ -26,4 +27,3 @@ do
 done
 
 curl --head --fail http://$CANVAS_HOST/health_check.json
-

build/new-jenkins/groovy/cache-migrations.groovy

@@ -1,94 +0,0 @@
-/*
- * Copyright (C) 2019 - present Instructure, Inc.
- *
- * This file is part of Canvas.
- *
- * Canvas is free software: you can redistribute it and/or modify it under
- * the terms of the GNU Affero General Public License as published by the Free
- * Software Foundation, version 3 of the License.
- *
- * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
- * details.
- *
- * You should have received a copy of the GNU Affero General Public License along
- * with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-def log(message) {
-  echo "[cache-migrations.groovy]: ${message}"
-}
-
-def prefix() { return "Canvas" }
-def postgresImage() { return  "$POSTGRES_CACHE_TAG" }
-def cassandraImage() { return "$CASSANDRA_CACHE_TAG" }
-def dynamodbImage() { return "$DYNAMODB_CACHE_TAG" }
-
-def cacheLoadFailed() {
-  def key = 'loaded'
-  return sh(
-    script: """\
-      [ ! -z "\$(docker images -q ${postgresImage()} 2> /dev/null)" ] \
-        && [ ! -z "\$(docker images -q ${cassandraImage()} 2> /dev/null)" ] \
-        && [ ! -z "\$(docker images -q ${dynamodbImage()} 2> /dev/null)" ] \
-        && echo "${key}"
-    """,
-    returnStdout: true
-  ).trim() != key
-}
-
-def commitMigratedImages() {
-  def postgresMessage = "Postgres migrated image for patchset $NAME"
-  sh "docker commit -m \"${postgresMessage}\" \$(docker ps -q --filter 'name=postgres_') ${postgresImage()}"
-
-  def cassandraMessage = "Cassandra migrated image for patchset $NAME"
-  sh "docker commit -m \"${cassandraMessage}\" \$(docker ps -q --filter 'name=cassandra_') ${cassandraImage()}"
-
-  def dynamodbMessage = "Dynamodb migrated image for patchset $NAME"
-  sh "docker commit -m \"${dynamodbMessage}\" \$(docker ps -q --filter 'name=dynamodb_') ${dynamodbImage()}"
-}
-
-def loadMigratedImages() {
-  dockerCacheLoad(image: postgresImage(), prefix: prefix())
-  dockerCacheLoad(image: cassandraImage(), prefix: prefix())
-  dockerCacheLoad(image: dynamodbImage(), prefix: prefix())
-}
-
-def pullAndBuild() {
-  sh 'build/new-jenkins/docker-compose-pull.sh'
-  sh 'docker-compose build'
-}
-
-def startAndMigrate() {
-  sh 'docker-compose up -d'
-  sh 'build/new-jenkins/docker-compose-setup-databases.sh'
-}
-
-def storeMigratedImages() {
-  dockerCacheStore(image: postgresImage(), prefix: prefix())
-  dockerCacheStore(image: cassandraImage(), prefix: prefix())
-  dockerCacheStore(image: dynamodbImage(), prefix: prefix())
-}
-
-// use cached images if available
-def createMigrateBuildUpCached() {
-  if (configuration.getBoolean('skip-cache')) {
-    log('Build cache is disabled! Ignoring any previously cached migrations and migrating from scratch.')
-    pullAndBuild()
-    startAndMigrate()
-  } else {
-    loadMigratedImages() // load images from docker cache
-
-    // detect if load was successfull, otherwise
-    if(cacheLoadFailed()) {
-      pullAndBuild() // build as normal
-    }
-
-    // migrate and commit images
-    startAndMigrate()
-    commitMigratedImages()
-  }
-}
-
-return this

build/new-jenkins/groovy/migrations.groovy

@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2020 - present Instructure, Inc.
+ *
+ * This file is part of Canvas.
+ *
+ * Canvas is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, version 3 of the License.
+ *
+ * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+cachedMigrationsHash = null
+cachedCassandraTag = null
+cachedDynamodbTag = null
+cachedPostgresTag = null
+MIGRATIONS_FILE = 'migrations.md5'
+
+def log(message) {
+  echo "[migrations.groovy]: ${message}"
+}
+
+def migrationHash() {
+  if(cachedMigrationsHash) {
+    return cachedMigrationsHash
+  } else if(!fileExists(MIGRATIONS_FILE)) {
+    try {
+      unstash name: "migrations_md5sum"
+    } catch(Exception ex) {
+      sh "build/new-jenkins/migrate-md5sum.sh > $MIGRATIONS_FILE"
+
+      stash name: "migrations_md5sum", includes: MIGRATIONS_FILE
+    }
+  }
+
+  cachedMigrationsHash = sh(
+    script: "cat $MIGRATIONS_FILE",
+    returnStdout: true
+  ).trim()
+
+  return cachedMigrationsHash
+}
+
+def imageMergeTag(name) {
+  return "${configuration.buildRegistryPath()}-$name-migrations:${configuration.gerritBranch()}-${migrationHash()}"
+}
+
+def imagePatchsetTag(name) {
+  return "${configuration.buildRegistryPath()}-$name-migrations:${imageTagVersion()}-${imageTag.suffix()}-${migrationHash()}"
+}
+
+def cassandraTag() {
+  return cachedCassandraTag
+}
+
+def dynamodbTag() {
+  return cachedDynamodbTag
+}
+
+def postgresTag() {
+  return cachedPostgresTag
+}
+
+def cacheLoadFailed() {
+  return sh(
+    script: """
+      DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect ${imageMergeTag('postgres')} && \
+      DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect ${imageMergeTag('cassandra')} && \
+      DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect ${imageMergeTag('dynamodb')}
+    """,
+    returnStatus: true
+  ) != 0
+}
+
+def runMigrations() {
+  if (configuration.getBoolean('skip-cache') || cacheLoadFailed() || configuration.isChangeMerged()) {
+    log('Ignoring any previously cached migrations and migrating from scratch.')
+
+    cachedCassandraTag = configuration.isChangeMerged() ? imageMergeTag('cassandra') : imagePatchsetTag('cassandra')
+    cachedDynamodbTag = configuration.isChangeMerged() ? imageMergeTag('dynamodb') : imagePatchsetTag('dynamodb')
+    cachedPostgresTag = configuration.isChangeMerged() ? imageMergeTag('postgres') : imagePatchsetTag('postgres')
+
+    sh 'build/new-jenkins/docker-compose-pull.sh'
+    sh 'build/new-jenkins/docker-compose-build-up.sh'
+    sh 'build/new-jenkins/docker-compose-setup-databases.sh'
+
+    def postgresMessage = "Postgres migrated image for MD5SUM ${migrationHash()}"
+    sh "docker commit -m \"${postgresMessage}\" \$(docker ps -q --filter 'name=postgres_') ${cachedPostgresTag}"
+    sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push ${cachedPostgresTag}"
+
+    def cassandraMessage = "Cassandra migrated image for MD5SUM ${migrationHash()}"
+    sh "docker commit -m \"${cassandraMessage}\" \$(docker ps -q --filter 'name=cassandra_') ${cachedCassandraTag}"
+    sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push ${cachedCassandraTag}"
+
+    def dynamodbMessage = "Dynamodb migrated image for MD5SUM ${migrationHash()}"
+    sh "docker commit -m \"${dynamodbMessage}\" \$(docker ps -q --filter 'name=dynamodb_') ${cachedDynamodbTag}"
+    sh "./build/new-jenkins/docker-with-flakey-network-protection.sh push ${cachedDynamodbTag}"
+  } else {
+    log('Continuing with previously cached migrations.')
+
+    cachedCassandraTag = imageMergeTag('cassandra')
+    cachedDynamodbTag = imageMergeTag('dynamodb')
+    cachedPostgresTag = imageMergeTag('postgres')
+  }
+}
+
+return this

build/new-jenkins/groovy/rspec.groovy

@@ -109,7 +109,6 @@ def _runRspecTestSuite(
         }
 
         sh 'build/new-jenkins/docker-compose-build-up.sh'
-        sh 'build/new-jenkins/docker-compose-setup-databases.sh'
         sh 'build/new-jenkins/docker-compose-rspec-parallel.sh'
       }
     }

build/new-jenkins/migrate-md5sum.sh

@@ -0,0 +1 @@
+find db/migrate/*.rb -type f -exec md5sum {} \; | sort -k 2 | md5sum | cut -d ' ' -f 1

build/new-jenkins/pact/docker-compose-pact-setup.sh

@@ -14,15 +14,5 @@ for service in cassandra:9160 redis:6379; do
   docker-compose exec -T canvas ./build/new-jenkins/wait-for-it ${service}
 done
 
-# create cassandra keyspaces later to be used by canvas
-create_cmd=""
-for keyspace in auditors global_lookups page_views; do
-  create_cmd+="CREATE KEYSPACE IF NOT EXISTS ${keyspace} WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };"
-done
-docker-compose exec -T cassandra cqlsh -e "${create_cmd[@]}"
-
-# migrate canvas test db
-docker-compose exec -T -e VERBOSE=false canvas bundle exec rails db:migrate
-
 # clone databases from canvas_test
 seq $((DOCKER_PROCESSES-1)) | parallel "docker-compose exec -T postgres sh -c 'createdb -U postgres -T canvas_test pact_test{}'"

docker-compose.new-jenkins.yml

@@ -38,6 +38,7 @@ services:
   postgres:
     image: $POSTGRES_IMAGE_TAG
     environment:
+      PGDATA: /data
       POSTGRES_PASSWORD: $POSTGRES_PASSWORD
     init: true