canvas-lms/Jenkinsfile.coverage

#!/usr/bin/env groovy

/*
 * Copyright (C) 2022 - present Instructure, Inc.
 *
 * This file is part of Canvas.
 *
 * Canvas is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, version 3 of the License.
 *
 * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License along
 * with this program. If not, see <http://www.gnu.org/licenses/>.
 */

library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
loadLocalLibrary('local-lib', 'build/new-jenkins/library')

@groovy.transform.Field
def rspecqNodeTotal = 23
@groovy.transform.Field
def rspecNodeTotal = 27

def setupNode() {
  sh 'rm -vrf ./tmp'

  checkout scm

  distribution.stashBuildScripts()

  credentials.withStarlordCredentials { ->
    sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
  }

  sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')
}

def getPatchsetTag() {
  (env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
}

def redisUrl() {
  return "redis://${TEST_QUEUE_HOST}:6379"
}

def generateSkippedSpecsReport() {
  try{
    copyArtifacts(
      filter: 'tmp/rspec_results/**',
      optional: false,
      projectName: env.JOB_NAME,
      selector: specific(env.BUILD_NUMBER),
    )

    withEnv(['COMPOSE_FILE=docker-compose.new-jenkins.yml']) {
      withCredentials([usernamePassword(credentialsId: 'INSENG_CANVAS_CI_AWS_ACCESS', usernameVariable: 'INSENG_AWS_ACCESS_KEY_ID', passwordVariable: 'INSENG_AWS_SECRET_ACCESS_KEY')]) {
        def awsCreds = "AWS_DEFAULT_REGION=us-west-2 AWS_ACCESS_KEY_ID=${INSENG_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${INSENG_AWS_SECRET_ACCESS_KEY}"
        sh "$awsCreds aws s3 cp s3://instructure-canvas-ci/skipped_specs_ruby.json skipped_specs.json"
        sh """
          docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/rspec_results/:/tmp/rspec_results \
          -v \$(pwd)/\$LOCAL_WORKDIR/skipped_specs.json/:/usr/src/app/skipped_specs.json \
          --name skipped-spec-collator canvas bash -c \
          "bundle install; ruby build/new-jenkins/skipped_specs_manager.rb ruby"
        """
        sh 'docker cp skipped-spec-collator:/tmp/skipped_specs.json skipped_specs.json'
        sh "$awsCreds aws s3 cp skipped_specs.json s3://instructure-canvas-ci/skipped_specs_ruby.json"
      }
      sendSkippedSpecsSlackReport()
      archiveArtifacts allowEmptyArchive: true, artifacts: 'skipped_specs.json'
    }
  } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
    slackSend channel: '#canvas-test-stats', color: 'danger', message: "<$env.BUILD_URL|coverage-ruby failed to generate skipped specs report!>"
  }
}

def sendSkippedSpecsSlackReport() {
  def rubySkippedSpecs = sh(script: "grep -o 'file_indicator' skipped_specs.json | wc -l", returnStdout: true).trim() ?: '0'
  def color = 'danger'
  if(rubySkippedSpecs.toInteger() < 100) {
    color = 'good'
  } else if (rubySkippedSpecs.toInteger() < 300) {
    color = 'warning'
  }
  def jobInfo = "<$env.BUILD_URL|Ruby>"
  slackSend channel: '#canvas-test-stats', color: color, message: "$rubySkippedSpecs skipped specs in $jobInfo! "
}

pipeline {
  agent none
  options {
    ansiColor('xterm')
    timeout(60)
    timestamps()
  }

  environment {
    COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
    FORCE_FAILURE = "${configuration.isForceFailureSelenium() ? '1' : ''}"
    RERUNS_RETRY = "${configuration.getInteger('rspecq-max-requeues')}"
    RSPECQ_FILE_SPLIT_THRESHOLD = "${configuration.fileSplitThreshold()}"
    RSPECQ_MAX_REQUEUES = "${configuration.getInteger('rspecq-max-requeues')}"
    SELENIUM_TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/selenium'
    TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
    EXCLUDE_TESTS = '.*/(selenium|contracts)'
    RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
    ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
    POSTGRES_PASSWORD = 'sekret'
    RSPECQ_REDIS_URL = redisUrl()
    PATCHSET_TAG = getPatchsetTag()

    CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
    DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
    POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')

    IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()
    RSPEC_PROCESSES = 6

    CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
    POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"

    COVERAGE_LOCATION = "${env.COVERAGE_TYPE == 'ruby-selenium' ? 'canvas__master__selenium--coverage/coverage' : (env.COVERAGE_TYPE == 'ruby-nonselenium' ? 'canvas__master__rspec--coverage/coverage' : 'canvas-lms-rspec/coverage')}"
  }

  stages {
    stage('Environment') {
      steps {
        script {
          def rspecNodeRequirements = [label: 'canvas-docker']

          def postBuildHandler = [
            onNodeReleasing: { stageName, stageConfig, result ->
              buildSummaryReport.saveRunManifest()

              copyArtifacts(
                filter: 'tmp/coverage/**',
                optional: false,
                projectName: env.JOB_NAME,
                selector: specific(env.BUILD_NUMBER),
              )

              withEnv(['COMPOSE_FILE=docker-compose.new-jenkins.yml']) {
                sh """
                  docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/coverage/:/tmp/coverage \
                  --name coverage-collator canvas bash -c \
                  "bundle install; bundle exec rake coverage:report['/tmp/coverage/canvas__*/**']"
                """

                sh 'docker cp coverage-collator:/usr/src/app/coverage/ coverage'

                archiveArtifacts allowEmptyArchive: true, artifacts: 'coverage/**'

                publishHTML target: [
                  allowMissing: false,
                  alwaysLinkToLastBuild: false,
                  keepAll: true,
                  reportDir: './coverage',
                  reportFiles: 'index.html',
                  reportName: 'Ruby Coverage Report'
                ]

                uploadCoverage([
                  uploadSource: '/coverage',
                  uploadDest: env.COVERAGE_LOCATION
                ])
              }
              if (env.GERRIT_EVENT_TYPE != 'comment-added' && env.COVERAGE_TYPE == 'ruby-total') {
                generateSkippedSpecsReport()
              }
            }
          ]

          def postStageHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
            }
          ]

          extendedStage('Runner').obeysAllowStages(false).execute {
            extendedStage('Builder').hooks(postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
              stage('Setup') {
                setupNode()
              }

              extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
                def rspecqStages = [:]

                if (env.COVERAGE_TYPE != 'ruby-selenium') {
                  extendedStage('RSpecQ Reporter for Rspec').timeout(30).queue(rspecqStages) {
                    try {
                      sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
                        --build=${JOB_NAME}_build${BUILD_NUMBER}_rspec \
                        --queue-wait-timeout 240 \
                        --redis-url $RSPECQ_REDIS_URL \
                        --report", label: 'Reporter')
                    } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
                      if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
                        /* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
                        sh '''#!/bin/bash
                          ids=( $(docker ps -aq --filter "name=canvas_") )
                          for i in "${ids[@]}"
                            do
                              docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
                          done
                        '''
                      }
                      throw e
                    }
                  }
                }

                if (env.COVERAGE_TYPE != 'ruby-nonselenium') {
                  extendedStage('RSpecQ Reporter for Selenium').timeout(30).queue(rspecqStages) {
                    try {
                      sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
                        --build=${JOB_NAME}_build${BUILD_NUMBER}_selenium \
                        --queue-wait-timeout 120 \
                        --redis-url $RSPECQ_REDIS_URL \
                        --report", label: 'Reporter')
                    } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
                      if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
                        /* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
                        sh '''#!/bin/bash
                          ids=($(docker ps -aq --filter "name=canvas_"))
                          for i in "${ids[@]}"
                            do
                              docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
                          done
                          '''
                      }
                      throw e
                    }
                  }

                  extendedStage('RSpecQ Selenium Set 00')
                      .envVars(['CI_NODE_INDEX=0',
                          "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_selenium",
                          "TEST_PATTERN=${env.SELENIUM_TEST_PATTERN}",
                          'EXCLUDE_TESTS=.*/(selenium/performance|instfs/selenium|contracts)'])
                      .hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('selenium') }])
                      .timeout(30)
                      .queue(rspecqStages) { rspecStage.runRspecqSuite() }

                  for (int i = 1; i < rspecqNodeTotal; i++) {
                    def index = i
                    extendedStage("RSpecQ Selenium Set ${(index).toString().padLeft(2, '0')}")
                      .envVars(["CI_NODE_INDEX=$index",
                          "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_selenium",
                          "TEST_PATTERN=${env.SELENIUM_TEST_PATTERN}",
                          'EXCLUDE_TESTS=.*/(selenium/performance|instfs/selenium|contracts)'])
                      .hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('selenium') }])
                      .nodeRequirements(rspecNodeRequirements)
                      .timeout(30)
                      .queue(rspecqStages) { rspecStage.runRspecqSuite() }
                  }
                }

                if (env.COVERAGE_TYPE != 'ruby-selenium') {
                  rspecNodeTotal.times { index ->
                    extendedStage("RSpecQ Rspec Set ${(index + 1).toString().padLeft(2, '0')}")
                      .envVars(["CI_NODE_INDEX=$index", "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}_rspec"])
                      .hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode('rspec') }])
                      .nodeRequirements(rspecNodeRequirements)
                      .timeout(30)
                      .queue(rspecqStages) { rspecStage.runRspecqSuite() }
                  }
                }

                parallel(rspecqStages)
              } //rspecQ
            } //builder
          } //runner
        } //script
      } //steps
    } //environment
  } //stages
} //pipeline