Revert "Crystalball post-merge map generation"
This reverts commit bd847c2d14.
Reason for revert: breaking cd
Change-Id: I85ed1dc69045d2cd984778da12c2e0b402b842ad
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/279637
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Brian Watson <bwatson@instructure.com>
Product-Review: Brian Watson <bwatson@instructure.com>
Reviewed-by: Andrea Cirulli <andrea.cirulli@instructure.com>
This commit is contained in:
Brian Watson
parent
34ec2324d6
commit
9a9c68be6e
|
|
@ -19,7 +19,6 @@
|
|||
/config/*.yml
|
||||
/config/credentials.yml.enc
|
||||
!/config/credentials.test.yml
|
||||
!/config/crystalball.yml
|
||||
/config/environments/*-local.rb
|
||||
/config/locales/generated/
|
||||
/config/RAILS6_1
|
||||
|
|
|
|||
|
|
@ -74,6 +74,4 @@ group :test do
|
|||
|
||||
# performance tools for instrumenting rspec tests
|
||||
gem "stackprof"
|
||||
|
||||
gem "crystalball", "0.7.0", require: false
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,140 +0,0 @@
|
|||
#!/usr/bin/env groovy
|
||||
|
||||
/*
|
||||
* Copyright (C) 2021 - present Instructure, Inc.
|
||||
*
|
||||
* This file is part of Canvas.
|
||||
*
|
||||
* Canvas is free software: you can redistribute it and/or modify it under
|
||||
* the terms of the GNU Affero General Public License as published by the Free
|
||||
* Software Foundation, version 3 of the License.
|
||||
*
|
||||
* Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
* A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License along
|
||||
* with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
library 'canvas-builds-library'
|
||||
loadLocalLibrary('local-lib', 'build/new-jenkins/library')
|
||||
|
||||
// if the build never starts or gets into a node block, then we
|
||||
// can never load a file. and a very noisy/confusing error is thrown.
|
||||
def ignoreBuildNeverStartedError(block) {
|
||||
try {
|
||||
block()
|
||||
}
|
||||
catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) {
|
||||
if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) {
|
||||
throw ex
|
||||
}
|
||||
else {
|
||||
echo "ignored MissingContextVariableException: \n${ex.message}"
|
||||
}
|
||||
// we can ignore this very noisy error
|
||||
}
|
||||
}
|
||||
|
||||
def getMigrationsTag(name) {
|
||||
(env.GERRIT_REFSPEC.contains('master')) || !migrations.cacheLoadFailed() ? migrations.imageMergeTag(name) : migrations.imagePatchsetTag(name)
|
||||
}
|
||||
|
||||
def getPatchsetTag() {
|
||||
(env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
|
||||
}
|
||||
|
||||
def getResultsHTMLUrl() {
|
||||
return "${env.BUILD_URL}/artifact/crystalball_map.yml"
|
||||
}
|
||||
|
||||
pipeline {
|
||||
agent { label 'canvas-docker' }
|
||||
options {
|
||||
ansiColor('xterm')
|
||||
timestamps()
|
||||
}
|
||||
|
||||
environment {
|
||||
BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
|
||||
POSTGRES = configuration.postgres()
|
||||
RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs
|
||||
// e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6
|
||||
PATCHSET_TAG = getPatchsetTag()
|
||||
|
||||
CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations')
|
||||
DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
|
||||
POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')
|
||||
|
||||
IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()
|
||||
RSPEC_PROCESSES = 6
|
||||
|
||||
CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
||||
DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
||||
POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES"
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Setup') {
|
||||
steps {
|
||||
cleanAndSetup()
|
||||
}
|
||||
}
|
||||
|
||||
stage('Parallel Run Tests') {
|
||||
steps {
|
||||
script {
|
||||
def stages = [:]
|
||||
|
||||
distribution.stashBuildScripts()
|
||||
rspecStage.createDistribution(stages)
|
||||
|
||||
parallel(stages)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
ignoreBuildNeverStartedError {
|
||||
node('master') {
|
||||
buildSummaryReport.publishReport('Build Summary Report', currentBuild.getResult() == 'SUCCESS' ? 'SUCCESS' : 'FAILURE')
|
||||
}
|
||||
}
|
||||
|
||||
copyArtifacts(
|
||||
filter: 'tmp/crystalball/**',
|
||||
optional: false,
|
||||
projectName: env.JOB_NAME,
|
||||
selector: specific(env.BUILD_NUMBER),
|
||||
)
|
||||
|
||||
sh """
|
||||
docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/crystalball/:/tmp/crystalball \
|
||||
-v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
|
||||
--name crystalball-parser \
|
||||
web bash -c 'ruby build/new-jenkins/crystalball_merge_coverage.rb /tmp/crystalball/'
|
||||
"""
|
||||
|
||||
sh 'docker cp crystalball-parser:/usr/src/app/crystalball_map.yml .'
|
||||
archiveArtifacts allowEmptyArchive: true, artifacts: 'crystalball_map.yml'
|
||||
|
||||
// Only alert on periodic jobs, not ones resulting from manual tests
|
||||
if (env.GERRIT_EVENT_TYPE != 'comment-added') {
|
||||
slackSend channel: '#crystalball-noisy', message: "<$env.BUILD_URL/testReport|Latest Crystalball Map Generated> - <${getResultsHTMLUrl()}|Map>"
|
||||
}
|
||||
}
|
||||
}
|
||||
cleanup {
|
||||
script {
|
||||
ignoreBuildNeverStartedError {
|
||||
libraryScript.execute 'bash/docker-cleanup.sh --allow-failure'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
#
|
||||
# Copyright (C) 2021 - present Instructure, Inc.
|
||||
#
|
||||
# This file is part of Canvas.
|
||||
#
|
||||
# Canvas is free software: you can redistribute it and/or modify it under
|
||||
# the terms of the GNU Affero General Public License as published by the Free
|
||||
# Software Foundation, version 3 of the License.
|
||||
#
|
||||
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License along
|
||||
# with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
|
||||
path = "/tmp/crystalball"
|
||||
map_header = nil
|
||||
map_body = {}
|
||||
Dir.glob("#{path}/**/*_map.yml") do |filename|
|
||||
puts "Looking through #{filename}"
|
||||
doc = File.read(filename)
|
||||
(header, body) = doc.split("---").reject(&:empty?)
|
||||
map_header ||= header
|
||||
body.split("\n").slice_when { |_before, after| after.include?(":") }.each do |group|
|
||||
spec = group.shift
|
||||
changed_files = group
|
||||
|
||||
next if spec.empty? || changed_files.count.zero?
|
||||
|
||||
raise "#{spec} already has entries: #{map_body[spec]}" unless map_body[spec].nil?
|
||||
|
||||
map_body[spec] = changed_files
|
||||
end
|
||||
end
|
||||
|
||||
File.open("crystalball_map.yml", "w") do |file|
|
||||
file << "---"
|
||||
file << map_header
|
||||
file << "---"
|
||||
file << "\n"
|
||||
map_body.each do |spec, app_files|
|
||||
file.puts spec
|
||||
file.puts app_files.join("\n")
|
||||
end
|
||||
end
|
||||
|
||||
puts "Crystalball Map Created for #{map_body.keys.count} tests!"
|
||||
|
|
@ -31,7 +31,6 @@ def createDistribution(nestedStages) {
|
|||
|
||||
def baseEnvVars = [
|
||||
"ENABLE_AXE_SELENIUM=${env.ENABLE_AXE_SELENIUM}",
|
||||
"ENABLE_CRYSTALBALL=${env.ENABLE_CRYSTALBALL}",
|
||||
'POSTGRES_PASSWORD=sekret',
|
||||
'SELENIUM_VERSION=3.141.59-20201119',
|
||||
"RSPECQ_ENABLED=${env.RSPECQ_ENABLED}"
|
||||
|
|
@ -147,14 +146,6 @@ def tearDownNode(prefix) {
|
|||
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results tmp/rspec_results canvas_ --allow-error --clean-dir'
|
||||
sh "build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures/$prefix canvas_ --allow-error --clean-dir"
|
||||
|
||||
// Don't generate map pre-merge
|
||||
if (env.ENABLE_CRYSTALBALL == '1' && env.RSPECQ_ENABLED != '1') {
|
||||
sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results/crystalball_results tmp/crystalball canvas_ --allow-error --clean-dir'
|
||||
sh 'ls tmp/crystalball'
|
||||
sh 'ls -R'
|
||||
archiveArtifacts allowEmptyArchive: true, artifacts: 'tmp/crystalball/**/*'
|
||||
}
|
||||
|
||||
if (configuration.getBoolean('upload-docker-logs', 'false')) {
|
||||
sh "docker ps -aq | xargs -I{} -n1 -P1 docker logs --timestamps --details {} 2>&1 > tmp/docker-${prefix}-${CI_NODE_INDEX}.log"
|
||||
archiveArtifacts(artifacts: "tmp/docker-${prefix}-${CI_NODE_INDEX}.log")
|
||||
|
|
@ -201,7 +192,6 @@ def runRspecqSuite() {
|
|||
return
|
||||
}
|
||||
sh(script: 'docker-compose exec -T -e ENABLE_AXE_SELENIUM \
|
||||
-e ENABLE_CRYSTALBALL \
|
||||
-e RSPECQ_ENABLED \
|
||||
-e SENTRY_DSN \
|
||||
-e RSPECQ_UPDATE_TIMINGS \
|
||||
|
|
@ -234,7 +224,7 @@ def runRspecqSuite() {
|
|||
|
||||
def runLegacySuite() {
|
||||
try {
|
||||
sh(script: 'docker-compose exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM -e ENABLE_CRYSTALBALL canvas bash -c \'build/new-jenkins/rspec-with-retries.sh\'', label: 'Run Tests')
|
||||
sh(script: 'docker-compose exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM canvas bash -c \'build/new-jenkins/rspec-with-retries.sh\'', label: 'Run Tests')
|
||||
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
|
||||
if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
|
||||
/* groovylint-disable-next-line GStringExpressionWithinString */
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
# Custom path to your execution map. Can be file or folder. Default: `tmp/crystalball_data.yml`
|
||||
execution_map_path: './crystalball_map.yml'
|
||||
# Maximum amount of examples which will be run automatically. Default: no limit.
|
||||
# If prediction size is over the limit Crystalball will prune prediction to fit the limit.
|
||||
# examples_limit: 1
|
||||
#
|
||||
# Custom prediction builder class to use. Default: 'Crystalball::RSpec::StandardPredictionBuilder'
|
||||
prediction_builder_class_name: 'Crystalball::RSpec::StandardPredictionBuilder'
|
||||
# #
|
||||
# Set of requires. Usually used to require custom predictor class file. Default: []
|
||||
requires:
|
||||
- './config/initializers/crystalball.rb'
|
||||
#
|
||||
# Custom map expiration period in seconds. 0 = Disable expiration check. Default: 86400 (1 day)
|
||||
map_expiration_period : 0
|
||||
#
|
||||
# Custom RSpec runner class to use. Default: 'Crystalball::RSpec::Runner'
|
||||
runner_class_name: 'Crystalball::RSpec::DryRunner'
|
||||
#
|
||||
# File path to save dry-run prediction results
|
||||
dry_run_output_file_path: './crystalball_spec_list.txt'
|
||||
#
|
||||
# "From" value for `git diff` command. default: 'HEAD'
|
||||
diff_from: 'HEAD'
|
||||
#
|
||||
# "To" value for `git diff` command. default: nil
|
||||
diff_to:
|
||||
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
#
|
||||
# Copyright (C) 2021 - present Instructure, Inc.
|
||||
#
|
||||
# This file is part of Canvas.
|
||||
#
|
||||
# Canvas is free software: you can redistribute it and/or modify it under
|
||||
# the terms of the GNU Affero General Public License as published by the Free
|
||||
# Software Foundation, version 3 of the License.
|
||||
#
|
||||
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License along
|
||||
# with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
require "rspec/core"
|
||||
require "crystalball/rspec/prediction_builder"
|
||||
require "crystalball/rspec/filtering"
|
||||
require "crystalball/rspec/prediction_pruning"
|
||||
|
||||
module Crystalball
|
||||
module RSpec
|
||||
# Our custom RSpec runner to generate and save predictions to a file, i.e. "dry-run"
|
||||
# TODO: make DryRunner class NOT inherit ::RSpec::Core::Runner because we don"t need it
|
||||
class DryRunner < ::RSpec::Core::Runner
|
||||
include PredictionPruning
|
||||
|
||||
class << self
|
||||
def run(args, err = $stderr, out = $stdout)
|
||||
return config["runner_class"].run(args, err, out) unless config["runner_class"] == self
|
||||
|
||||
Crystalball.log :info, "Crystalball starts to glow..."
|
||||
prediction = build_prediction
|
||||
dry_run(prediction) if args.include?("--dry-run")
|
||||
|
||||
Crystalball.log :debug, "Prediction: #{prediction.first(5).join(" ")}#{"..." if prediction.size > 5}"
|
||||
Crystalball.log :info, "Starting RSpec."
|
||||
|
||||
super(args + prediction, err, out)
|
||||
end
|
||||
|
||||
def dry_run(prediction)
|
||||
prediction_file_path = config["dry_run_output_file_path"]
|
||||
File.write(prediction_file_path, prediction.to_a.join(","))
|
||||
Crystalball.log :info, "Saved RSpec prediction to #{prediction_file_path}"
|
||||
exit # rubocop:disable Rails/Exit
|
||||
end
|
||||
|
||||
def reset!
|
||||
self.prediction_builder = nil
|
||||
self.config = nil
|
||||
end
|
||||
|
||||
def prepare
|
||||
config["runner_class"].load_execution_map
|
||||
end
|
||||
|
||||
def prediction_builder
|
||||
@prediction_builder ||= config["prediction_builder_class"].new(config)
|
||||
end
|
||||
|
||||
def config
|
||||
@config ||= begin
|
||||
config_src = if config_file
|
||||
require "yaml"
|
||||
YAML.safe_load(config_file.read)
|
||||
else
|
||||
{}
|
||||
end
|
||||
|
||||
Crystalball::RSpec::Runner::Configuration.new(config_src)
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def load_execution_map
|
||||
check_map
|
||||
prediction_builder.execution_map
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_writer :config, :prediction_builder
|
||||
|
||||
def config_file
|
||||
file = Pathname.new(ENV.fetch("CRYSTALBALL_CONFIG", "crystalball.yml"))
|
||||
file = Pathname.new("config/crystalball.yml") unless file.exist?
|
||||
file.exist? ? file : nil
|
||||
end
|
||||
|
||||
def build_prediction
|
||||
check_map
|
||||
prune_prediction_to_limit(prediction_builder.prediction.sort_by(&:length))
|
||||
end
|
||||
|
||||
def check_map
|
||||
Crystalball.log :warn, "Maps are outdated!" if prediction_builder.expired_map?
|
||||
end
|
||||
end
|
||||
|
||||
def setup(err, out)
|
||||
configure(err, out)
|
||||
@configuration.load_spec_files
|
||||
|
||||
Filtering.remove_unnecessary_filters(@configuration, @options.options[:files_or_directories_to_run])
|
||||
|
||||
if reconfiguration_needed?
|
||||
Crystalball.log :warn, "Prediction examples size #{@world.example_count} is over the limit (#{examples_limit})"
|
||||
Crystalball.log :warn, "Prediction is pruned to fit the limit!"
|
||||
|
||||
reconfigure_to_limit
|
||||
@configuration.load_spec_files
|
||||
end
|
||||
|
||||
@world.announce_filters
|
||||
end
|
||||
|
||||
# Backward compatibility for RSpec < 3.7
|
||||
def configure(err, out)
|
||||
@configuration.error_stream = err
|
||||
@configuration.output_stream = out if @configuration.output_stream == $stdout
|
||||
@options.configure(@configuration)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
require "crystalball/rspec/runner/configuration"
|
||||
|
|
@ -32,7 +32,6 @@ end
|
|||
|
||||
require "securerandom"
|
||||
require "tmpdir"
|
||||
require "crystalball"
|
||||
|
||||
ENV["RAILS_ENV"] = "test"
|
||||
require_relative "../config/environment"
|
||||
|
|
@ -157,28 +156,6 @@ if ENV["ENABLE_AXE_SELENIUM"] == "1"
|
|||
end
|
||||
end
|
||||
|
||||
# Don't do map generation in pre-merge, which runs rspecq
|
||||
if ENV["ENABLE_CRYSTALBALL"] == "1" && ENV["RSPECQ_ENABLED"] != "1"
|
||||
Crystalball::MapGenerator.start! do |config|
|
||||
config.register Crystalball::MapGenerator::CoverageStrategy.new
|
||||
config.map_storage_path = "log/results/crystalball_results/#{ENV.fetch("PARALLEL_INDEX", "0")}_map.yml"
|
||||
end
|
||||
end
|
||||
|
||||
module Crystalball
|
||||
class MapGenerator
|
||||
class CoverageStrategy
|
||||
def call(example_map, example)
|
||||
puts "Calling Coverage Strategy for #{example.inspect}"
|
||||
before = Coverage.peek_result
|
||||
yield example_map, example
|
||||
after = Coverage.peek_result
|
||||
example_map.push(*execution_detector.detect(before, after))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module RSpec::Rails
|
||||
module ViewExampleGroup
|
||||
module ExampleMethods
|
||||
|
|
|
|||
Loading…
Reference in New Issue