canvas-lms/Jenkinsfile.crystalball

#!/usr/bin/env groovy

/*
 * Copyright (C) 2021 - present Instructure, Inc.
 *
 * This file is part of Canvas.
 *
 * Canvas is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, version 3 of the License.
 *
 * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License along
 * with this program. If not, see <http://www.gnu.org/licenses/>.
 */

library 'canvas-builds-library'
loadLocalLibrary('local-lib', 'build/new-jenkins/library')

commitMessageFlag.setDefaultValues(commitMessageFlagDefaults() + commitMessageFlagPrivateDefaults())

@groovy.transform.Field
def rspecqNodeTotal = 50

// if the build never starts or gets into a node block, then we
// can never load a file. and a very noisy/confusing error is thrown.
def ignoreBuildNeverStartedError(block) {
  try {
    block()
  }
  catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) {
    if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) {
      throw ex
    }
    else {
      echo "ignored MissingContextVariableException: \n${ex.message}"
    }
    // we can ignore this very noisy error
  }
}

def getMigrationsTag(name) {
  (env.GERRIT_REFSPEC.contains('master')) || !migrations.cacheLoadFailed() ? migrations.imageMergeTag(name) : migrations.imagePatchsetTag(name)
}

def getPatchsetTag() {
  (env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
}

def getResultsHTMLUrl() {
  return "${env.BUILD_URL}/artifact/crystalball_map.yml"
}

def postFn() {
  copyArtifacts(
    filter: 'tmp/*/crystalball/**',
    optional: false,
    projectName: env.JOB_NAME,
    selector: specific(env.BUILD_NUMBER),
  )

  sh """
    docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/:/tmp \
    -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
    --name crystalball-parser \
    canvas bash -c 'ruby build/new-jenkins/crystalball_merge_coverage.rb "/tmp/*/crystalball/"'
  """

  sh 'docker cp crystalball-parser:/usr/src/app/crystalball_map.yml .'
  archiveArtifacts allowEmptyArchive: true, artifacts: 'crystalball_map.yml'

  def message = "<$env.BUILD_URL/testReport|Latest Crystalball Map Generated> - <${getResultsHTMLUrl()}|Map>\n"
  try {
    def mapSpecInfo = sh(script: """
      docker-compose run --rm \
        -v \$(pwd)/\$LOCAL_WORKDIR/crystalball_map.yml/:/usr/src/app/crystalball_map.yml \
        -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
        -v \$(pwd)/\$LOCAL_WORKDIR/gems/plugins/:/usr/src/app/gems/plugins \
        -v \$(pwd)/\$LOCAL_WORKDIR/spec:/usr/src/app/spec \
        canvas bash -c 'ruby build/new-jenkins/crystalball_map_smoke_test.rb'
      """
    , returnStdout: true)
    message = message + "\n" + mapSpecInfo
    // Only alert and push to s3 on periodic jobs, not ones resulting from manual tests
    if (env.CRYSTALBALL_MAP_PUSH_TO_S3 == '1' && env.GERRIT_EVENT_TYPE != 'comment-added') {
      sh 'aws s3 cp crystalball_map.yml s3://instructure-canvas-ci/'
    }
  } catch(e) {
    message = message + "\nMap Invalid!"
  } finally {
    echo message
    slackSend channel: '#crystalball-noisy', message: message
  }
}

pipeline {
  agent none
  options {
    ansiColor('xterm')
    timestamps()
  }

  environment {
    BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
    COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
    COMPOSE_PROJECT_NAME = 'crystalball-map'
    RERUNS_RETRY = commitMessageFlag('rspecq-max-requeues').asType(Integer)
    RSPECQ_FILE_SPLIT_THRESHOLD = '9999'
    RSPECQ_MAX_REQUEUES = commitMessageFlag('rspecq-max-requeues').asType(Integer)
    RSPEC_PROCESSES = 6
    TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
    EXCLUDE_TESTS = '.*/(selenium/performance|instfs/selenium|contracts)'
    POSTGRES_PASSWORD = 'sekret'
    POSTGRES = configuration.postgres()
    RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
    // e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6
    PATCHSET_TAG = getPatchsetTag()

    BASE_RUNNER_PREFIX = configuration.buildRegistryPath('base-runner')
    CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
    DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
    KARMA_RUNNER_PREFIX = configuration.buildRegistryPath('karma-runner')
    LINTERS_RUNNER_PREFIX = configuration.buildRegistryPath('linters-runner')
    POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')
    RUBY_RUNNER_PREFIX = configuration.buildRegistryPath('ruby-runner')
    YARN_RUNNER_PREFIX = configuration.buildRegistryPath('yarn-runner')
    WEBPACK_BUILDER_PREFIX = configuration.buildRegistryPath('webpack-builder')
    WEBPACK_ASSETS_PREFIX = configuration.buildRegistryPath('webpack-assets')

    IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()

    CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    POSTGRES_CLIENT = configuration.postgresClient()
  }

  stages {
    stage('Environment') {
      steps {
        script {
          def rspecNodeRequirements = [label: 'canvas-docker']

          def postBuildHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              ignoreBuildNeverStartedError {
                node('master') {
                  buildSummaryReport.publishReport('Build Summary Report', stageConfig.status())
                }
              }

              buildSummaryReport.addFailureRun('Main Build', currentBuild)
            },
            onNodeReleasing: {
              postFn()
            }
          ]

          def postStageHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
            }
          ]

          extendedStage('Root').hooks(postBuildHandler).obeysAllowStages(false).timeout(60).reportTimings(false).nodeRequirements(rspecNodeRequirements).execute {
            stage('Clean and Setup') {
              cleanAndSetup()
              setupStage()
              distribution.stashBuildScripts()
            }

            extendedStage('Build Docker Image')
              .hooks(buildSummaryReportHooks.call())
              .obeysAllowStages(false)
              .timeout(20)
              .execute{ buildDockerImageStage.patchsetImage() }

            extendedStage('Run Migrations')
              .hooks(buildSummaryReportHooks.call())
              .obeysAllowStages(false)
              .timeout(10)
              .execute { runMigrationsStage() }

            extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
              def rspecqStages = [:]

              // Pull docker images before running Reporter
              // sometimes it takes longer than usual to pull from starlord causing Reporter to timeout
              credentials.withStarlordCredentials { ->
                sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
              }

              extendedStage('RSpecQ Reporter').timeout(30).queue(rspecqStages) {
                try {
                  sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
                    --build=${JOB_NAME}_build${BUILD_NUMBER} \
                    --queue-wait-timeout 120 \
                    --redis-url $RSPECQ_REDIS_URL \
                    --report", label: 'Reporter')
                } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
                  if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
                    /* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
                    sh '''#!/bin/bash
                      ids=($(docker ps -aq --filter "name=canvas_"))
                      for i in "${ids[@]}"
                        do
                          docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
                      done
                    '''
                  }
                  throw e
                }
              }

              extendedStage('RSpecQ Set 00')
                  .envVars(['CI_NODE_INDEX=0',
                      "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}",
                      "CRYSTALBALL_MAP=1"])
                  .hooks(postStageHandler + [onNodeAcquired: { sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers') }, onNodeReleasing: { rspecStage.tearDownNode() }])
                  .timeout(30)
                  .queue(rspecqStages) {
                    rspecStage.runRspecqSuite()
                  }

              for (int i = 1; i < rspecqNodeTotal; i++) {
                def index = i
                extendedStage("RSpecQ Set ${(index).toString().padLeft(2, '0')}")
                  .envVars(["CI_NODE_INDEX=$index",
                      "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}",
                      "CRYSTALBALL_MAP=1"])
                  .hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }])
                  .nodeRequirements(rspecNodeRequirements)
                  .timeout(30)
                  .queue(rspecqStages) { rspecStage.runRspecqSuite() }
              }

              parallel(rspecqStages)
            } //rspecQ
          }//root
        }//script
      }//steps
    }//stage environment
  }//stages
}//pipeline