canvas-lms/Jenkinsfile.rspecq

#!/usr/bin/env groovy

/*
 * Copyright (C) 2022 - present Instructure, Inc.
 *
 * This file is part of Canvas.
 *
 * Canvas is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, version 3 of the License.
 *
 * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License along
 * with this program. If not, see <http://www.gnu.org/licenses/>.
 */

library "canvas-builds-library@${env.CANVAS_BUILDS_REFSPEC}"
loadLocalLibrary('local-lib', 'build/new-jenkins/library')

commitMessageFlag.setDefaultValues(commitMessageFlagDefaults() + commitMessageFlagPrivateDefaults())

@groovy.transform.Field
def rspecqNodeTotal = 50
@groovy.transform.Field
def rspecqTargetTime = 180
@groovy.transform.Field
def summaryMessage = ''
@groovy.transform.Field
def crystalballColor = 'danger'
@groovy.transform.Field
def specUnique = []

def computeTestCountNecessary() {
  copyArtifacts filter: 'tmp/crystalball_spec_list.txt', projectName: "${env.UPSTREAM}", selector: upstream()
  copyArtifacts filter: 'tmp/crystalball_map_version.txt', projectName: "${env.UPSTREAM}", selector: upstream()

  def specs = readFile(file: 'tmp/crystalball_spec_list.txt')
  def mapVersion = readFile(file: 'tmp/crystalball_map_version.txt')
  specUnique = specs.tokenize(',').unique()
  env.CRYSTALBALL_SPEC = specUnique.join(' ')

  summaryMessage = "Map: $mapVersion\n"

  if (commitMessageFlag('skip-crystalball') as Boolean) {
    summaryMessage += "Skip Crystalball Detected! - Running everything!"
    crystalballColor = 'danger'
    env.CRYSTALBALL_SPEC = '.'
    return false
  }

  // Crystalball predictor returned empty text file
  if (specUnique.size() == 0) {
    summaryMessage += "Configuration Changes Detected! - Running everything!"
    crystalballColor = 'danger'
    env.CRYSTALBALL_SPEC = '.'
    return false
  }
  // Crystalball predictor returned "."
  if (specUnique.size() == 1 && specUnique[0] == '.') {
    summaryMessage += "New File Detected! - Complete Suite Re-run!"
    crystalballColor = 'danger'
    env.CRYSTALBALL_SPEC = '.'
    return false
  }

  return true
}

def computeTestCount() {
  credentials.withStarlordCredentials { ->
    sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images')
  }

  sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers')

  sh(script: "docker-compose exec -T canvas bundle exec rspec --dry-run \
                                      --require './spec/formatters/node_count_formatter.rb' \
                                      --format NodeCountRecorder \
                                      --out formatter_out.txt --pattern ${specUnique.join(',')}", label: 'Get Node Count')

  def formatterResult = sh(script: 'docker-compose exec -T canvas cat formatter_out.txt', returnStdout: true).trim().split(" ")

  def nodeTotal = formatterResult[0].toInteger()
  def specTotal = formatterResult[1].toInteger()

  summaryMessage += "${specTotal} specs across ${specUnique.size()} files using ${nodeTotal} nodes"

  echo "=== Prediction Summary ===\n${summaryMessage}\n${specUnique.join('\n')}"

  crystalballColor = nodeTotal > rspecqNodeTotal ? 'warning' : 'good'
  rspecqNodeTotal = nodeTotal > rspecqNodeTotal ? rspecqNodeTotal : nodeTotal

  def jqAddCommand = []

  specUnique.each { x ->
    def globMatcher = x =~ /^(.+)\*spec\.rb$/
    if (globMatcher.matches()) {
      jqAddCommand.add("([with_entries(select(.key | match(\"${globMatcher.group(1)}\"))) | to_entries[] | .value] | add)")
    } else {
      jqAddCommand.add(".[\"./${x}\"]")
    }
  }

  def specTimings = sh(script: """#!/usr/bin/env bash
    set -ex

    docker run \
      -e TEST_QUEUE_HOST \
      -v \$(pwd)/build/new-jenkins/iterscores.lua:/iterscores.lua \
      -t \
      --rm \
      $REGISTRY_BASE/redis:alpine /bin/sh -c 'redis-cli -h $TEST_QUEUE_HOST -p 6379 --eval /iterscores.lua' | jq 'fromjson | ${jqAddCommand.join(' + ')}'
  """, returnStdout: true)

  if (specTimings && specTimings.isNumber()) {
    echo "=== Total Spec Time $specTimings"

    def newNodeTotal = Math.ceil(specTimings.toFloat() / (rspecqTargetTime * RSPEC_PROCESSES.toInteger()))

    summaryMessage += " (new: ${newNodeTotal})"
  } else {
    echo "=== No Spec Timings Found! ==="
  }
}

def sendCrystalballSlack(summary, color, status) {
  def jobInfo = "<https://gerrit.instructure.com/$env.GERRIT_CHANGE_NUMBER|Gerrit> | <$env.BUILD_URL|Jenkins>: **$status!**"
  def message = "$jobInfo\n$summary"
  slackSend channel: '#crystalball-noisy', color: color, message: message
}

def sendCrystalballMetrics() {
    def queueInfo = sh(script: "docker run -e TEST_QUEUE_HOST -t --rm $REGISTRY_BASE/redis:alpine /bin/sh -c '\
                                          redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:example_count;\
                                          redis-cli -h $TEST_QUEUE_HOST -p 6379 get ${JOB_NAME}_build${BUILD_NUMBER}:build_time'", returnStdout: true).split('\n')
    def exampleCount = queueInfo[0].replaceAll('"', '').trim()
    def buildTime = queueInfo[1].replaceAll('"', '').trim()

    reportToSplunk('rspecq_crystalball_data', [
                    'node_count': rspecqNodeTotal,
                    'example_count': exampleCount.toInteger(),
                    'execution_time': buildTime.toInteger(),
                    'result': currentBuild.currentResult,
                    'upstream_tag': "${env.UPSTREAM_TAG}"])
}

def redisUrl() {
  return "redis://${TEST_QUEUE_HOST}:6379"
}

pipeline {
  agent none
  options {
    ansiColor('xterm')
    timeout(60)
    timestamps()
  }

  environment {
    REGISTRY_BASE = 'starlord.inscloudgate.net/jenkins'
    COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
    COMPOSE_PROJECT_NAME = 'test-queue'
    FORCE_FAILURE = commitMessageFlag("force-failure-rspec").asBooleanInteger()
    RERUNS_RETRY = commitMessageFlag('rspecq-max-requeues').asType(Integer)
    RSPEC_PROCESSES = commitMessageFlag('rspecq-processes').asType(Integer)
    RSPECQ_FILE_SPLIT_THRESHOLD = commitMessageFlag('rspecq-file-split-threshold').asType(Integer)
    RSPECQ_MAX_REQUEUES = commitMessageFlag('rspecq-max-requeues').asType(Integer)
    TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
    EXCLUDE_TESTS = '.*/(selenium/performance|instfs/selenium|contracts)'
    RSPECQ_UPDATE_TIMINGS = "${env.GERRIT_EVENT_TYPE == 'change-merged' ? '1' : '0'}"
    ENABLE_AXE_SELENIUM = "${env.ENABLE_AXE_SELENIUM}"
    POSTGRES_PASSWORD = 'sekret'
    RSPECQ_REDIS_URL = redisUrl()
  }

  stages {
    stage('Environment') {
      steps {
        script {
          def rspecNodeRequirements = [label: nodeLabel()]
          env.CRYSTALBALL_SPEC = '.'

          def postRunnerHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              node('master') {
                if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
                  sendCrystalballSlack(summaryMessage, crystalballColor, stageConfig.status())
                }
                buildSummaryReport.saveRunManifest()
              }
            }
          ]

          def postBuildHandler = [
            onNodeReleasing: {
              if (!configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1') {
                sendCrystalballMetrics()
              }
            }
          ]

          def postStageHandler = [
            onStageEnded: { stageName, stageConfig, result ->
              buildSummaryReport.setStageTimings(stageName, stageConfig.timingValues())
            }
          ]

          extendedStage('Runner').hooks(postRunnerHandler).obeysAllowStages(false).execute {
            extendedStage('Builder').hooks(postStageHandler + postBuildHandler).obeysAllowStages(false).nodeRequirements(rspecNodeRequirements).execute {
              extendedStage('RSpecQ Setup').hooks(postStageHandler).obeysAllowStages(false).execute {
                sh 'rm -vrf ./tmp'

                checkout scm

                distribution.stashBuildScripts()
              }

              def testCountNecessary = !configuration.isChangeMerged() && env.GERRIT_REFSPEC != "refs/heads/master" && env.ENABLE_CRYSTALBALL == '1' && computeTestCountNecessary()

              if (testCountNecessary) {
                extendedStage('RSpecQ Compute Build Distribution').hooks(postStageHandler).obeysAllowStages(false).execute {

                  computeTestCount()
                }
              }

              extendedStage('Parallel Run Tests').obeysAllowStages(false).execute { stageConfig, buildConfig ->
                def rspecqStages = [:]

                def initialNodeHooks = testCountNecessary ?
                  postStageHandler + [onNodeReleasing: { rspecStage.tearDownNode() }] :
                  postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }]

                extendedStage('RSpecQ Initial Node')
                    .envVars(['CI_NODE_INDEX=0',
                        "CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
                        "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
                    .hooks(initialNodeHooks)
                    .timeout(env.NODE_TIMEOUT.toInteger())
                    .queue(rspecqStages) {
                      def initialNodeStages = [:]

                      extendedStage('RSpecQ Reporter').queue(initialNodeStages) {
                        try {
                          sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $PATCHSET_TAG bundle exec rspecq \
                            --build=${JOB_NAME}_build${BUILD_NUMBER} \
                            --queue-wait-timeout 120 \
                            --redis-url $RSPECQ_REDIS_URL \
                            --report", label: 'Reporter')
                        } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
                          if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
                            /* groovylint-disable-next-line GStringExpressionWithinString, SpaceInsideParentheses */
                            sh '''#!/bin/bash
                              ids=($(docker ps -aq --filter "name=canvas_"))
                              for i in "${ids[@]}"
                                do
                                  docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
                              done
                            '''
                          }
                          throw e
                        }
                      }

                      extendedStage("RSpecQ Set 00").queue(initialNodeStages) {
                        rspecStage.runRspecqSuite()
                      }

                      parallel(initialNodeStages)
                    }

                for (int i = 1; i < rspecqNodeTotal; i++) {
                  def index = i
                  extendedStage("RSpecQ Set ${(index).toString().padLeft(2, '0')}")
                    .envVars(["CI_NODE_INDEX=$index",
                        "CRYSTAL_BALL_SPECS=${env.CRYSTALBALL_SPEC}",
                        "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}"])
                    .hooks(postStageHandler + [onNodeAcquired: { rspecStage.setupNode() }, onNodeReleasing: { rspecStage.tearDownNode() }])
                    .nodeRequirements(rspecNodeRequirements)
                    .timeout(env.NODE_TIMEOUT.toInteger())
                    .queue(rspecqStages) { rspecStage.runRspecqSuite() }
                }

                parallel(rspecqStages)
              } //rspecQ
            } //builder
          } //runner
        } //script
      } //steps
    } //environment
  } //stages
} //pipeline