canvas-lms/Jenkinsfile.crystalball

#!/usr/bin/env groovy

/*
 * Copyright (C) 2021 - present Instructure, Inc.
 *
 * This file is part of Canvas.
 *
 * Canvas is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, version 3 of the License.
 *
 * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License along
 * with this program. If not, see <http://www.gnu.org/licenses/>.
 */

library 'canvas-builds-library'
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
final static RUN_MIGRATIONS_STAGE = 'Run Migrations'

// if the build never starts or gets into a node block, then we
// can never load a file. and a very noisy/confusing error is thrown.
def ignoreBuildNeverStartedError(block) {
  try {
    block()
  }
  catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) {
    if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) {
      throw ex
    }
    else {
      echo "ignored MissingContextVariableException: \n${ex.message}"
    }
    // we can ignore this very noisy error
  }
}

def getMigrationsTag(name) {
  (env.GERRIT_REFSPEC.contains('master')) || !migrations.cacheLoadFailed() ? migrations.imageMergeTag(name) : migrations.imagePatchsetTag(name)
}

def getPatchsetTag() {
  (env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
}

def getResultsHTMLUrl() {
  return "${env.BUILD_URL}/artifact/crystalball_map.yml"
}

pipeline {
  agent { label 'canvas-docker' }
  options {
    ansiColor('xterm')
    timestamps()
  }

  environment {
    BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
    POSTGRES = configuration.postgres()
    RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
    // e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6
    PATCHSET_TAG = getPatchsetTag()

    BASE_RUNNER_PREFIX = configuration.buildRegistryPath('base-runner')
    CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
    DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
    KARMA_BUILDER_PREFIX = configuration.buildRegistryPath('karma-builder')
    KARMA_RUNNER_PREFIX = configuration.buildRegistryPath('karma-runner')
    LINTERS_RUNNER_PREFIX = configuration.buildRegistryPath('linters-runner')
    POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')
    RUBY_RUNNER_PREFIX = configuration.buildRegistryPath('ruby-runner')
    YARN_RUNNER_PREFIX = configuration.buildRegistryPath('yarn-runner')
    WEBPACK_BUILDER_PREFIX = configuration.buildRegistryPath('webpack-builder')
    WEBPACK_CACHE_PREFIX = configuration.buildRegistryPath('webpack-cache')

    IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()
    RSPEC_PROCESSES = 6

    CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    POSTGRES_CLIENT = configuration.postgresClient()
  }

  stages {
    stage('Pre-Setup') {
      steps {
        cleanAndSetup()
      }
    }

    stage('Crystalball Map') {
      steps {
        script {
          def postBuildHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              buildSummaryReport.addFailureRun('Main Build', currentBuild)
            }
          ]

          distribution.stashBuildScripts()
          extendedStage('Root').hooks(postBuildHandler).obeysAllowStages(false).reportTimings(false).execute {
            def rootStages = [:]

            extendedStage('Builder').nodeRequirements(label: 'canvas-docker', podTemplate: null).obeysAllowStages(false).reportTimings(false).queue(rootStages) {
              extendedStage('Setup')
                .hooks(buildSummaryReportHooks.call())
                .obeysAllowStages(false)
                .timeout(2)
                .execute { setupStage() }

              extendedStage('Build Docker Image (Pre-Merge)')
                .hooks(buildSummaryReportHooks.call())
                .obeysAllowStages(false)
                .required(configuration.isChangeMerged())
                .timeout(20)
                .execute(buildDockerImageStage.&premergeCacheImage)

              extendedStage('Build Docker Image')
                .hooks(buildSummaryReportHooks.call())
                .obeysAllowStages(false)
                .timeout(20)
                .execute(buildDockerImageStage.&patchsetImage)

              extendedStage(RUN_MIGRATIONS_STAGE)
                .hooks(buildSummaryReportHooks.call())
                .obeysAllowStages(false)
                .timeout(10)
                .execute { runMigrationsStage() }
            }

            extendedStage("${RUN_MIGRATIONS_STAGE} (Waiting for Dependencies)").obeysAllowStages(false).waitsFor(RUN_MIGRATIONS_STAGE, 'Builder').queue(rootStages) { stageConfig, buildConfig ->
              def nestedStages = [:]
              rspecStage.createDistribution(nestedStages)

              parallel(nestedStages)
            }

            parallel(rootStages)
          }
        }
      }
    }
  }

  post {
    always {
      script {
        ignoreBuildNeverStartedError {
          node('master') {
            buildSummaryReport.publishReport('Build Summary Report', currentBuild.getResult() == 'SUCCESS' ? 'SUCCESS' : 'FAILURE')
          }
        }

        copyArtifacts(
          filter: 'tmp/crystalball/**',
          optional: false,
          projectName: env.JOB_NAME,
          selector: specific(env.BUILD_NUMBER),
        )

        sh """
          docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/crystalball/:/tmp/crystalball \
          -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
          --name crystalball-parser \
          web bash -c 'ruby build/new-jenkins/crystalball_merge_coverage.rb /tmp/crystalball/'
        """

        sh 'docker cp crystalball-parser:/usr/src/app/crystalball_map.yml .'
        archiveArtifacts allowEmptyArchive: true, artifacts: 'crystalball_map.yml'

        def message = "<$env.BUILD_URL/testReport|Latest Crystalball Map Generated> - <${getResultsHTMLUrl()}|Map>\n"
        try {
          def mapSpecInfo = sh(script: """
            docker-compose run --rm \
              -v \$(pwd)/\$LOCAL_WORKDIR/crystalball_map.yml/:/usr/src/app/crystalball_map.yml \
              -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
              -v \$(pwd)/\$LOCAL_WORKDIR/gems/plugins/:/usr/src/app/gems/plugins \
              -v \$(pwd)/\$LOCAL_WORKDIR/spec:/usr/src/app/spec \
              web bash -c 'ruby build/new-jenkins/crystalball_map_smoke_test.rb'
            """
          , returnStdout: true)
          message = message + "\n" + mapSpecInfo
          // Only alert and push to s3 on periodic jobs, not ones resulting from manual tests
          if (env.CRYSTALBALL_MAP_PUSH_TO_S3 == '1' && env.GERRIT_EVENT_TYPE != 'comment-added') {
            sh 'aws s3 cp crystalball_map.yml s3://instructure-canvas-ci/'
          }
        } catch(e) {
          message = message + "\nMap Invalid!"
        } finally {
          echo message
          slackSend channel: '#crystalball-noisy', message: message
        }
      }
    }
    cleanup {
      script {
        ignoreBuildNeverStartedError {
          libraryScript.execute 'bash/docker-cleanup.sh --allow-failure'
        }
      }
    }
  }
}