From 347f73b172fcd43218dd8e72c54173e4602a266d Mon Sep 17 00:00:00 2001 From: Kyle Rosenbaum Date: Mon, 26 Apr 2021 14:28:15 -0700 Subject: [PATCH] remove docker_processes from core rspec and selenium test runs refs DE-467 it is no longer possible to spin up multiple docker containers ona single rspec or selenium agent when running tests. parallelization is now exclusively done through multple rspec processes on a single container. test plan: - setting docker_processes ENV variable has no effect - settting rspec_processes still runs multiple test threads - tests continue to run as expected given that docker_processes has been set to 1 for multiple months. Change-Id: I604d953a71ad220373b6a71d558f46bff29f820e Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/263701 Tested-by: Service Cloud Jenkins Reviewed-by: James Butters QA-Review: James Butters Product-Review: James Butters --- Jenkinsfile.contract-tests | 1 - Jenkinsfile.selenium.flakey_spec_catcher | 3 +-- build/new-jenkins/docker-compose-build-up.sh | 11 +++-------- .../docker-compose-rspec-parallel.sh | 7 ------- ...docker-compose-setup-cassandra-keyspaces.sh | 14 -------------- .../docker-compose-setup-databases.sh | 18 +++++++----------- build/new-jenkins/library/vars/rspec.groovy | 6 +----- build/new-jenkins/rspec-flakey-spec-catcher.sh | 2 +- build/new-jenkins/rspec-tests.sh | 2 +- build/new-jenkins/rspec-with-retries.sh | 4 ++-- build/new-jenkins/test-engines.sh | 2 +- build/new-jenkins/test-gems.sh | 2 +- build/new-jenkins/xbrowser-test.sh | 2 +- docker-compose.new-jenkins.yml | 2 +- spec/coverage_tool.rb | 2 +- spec/spec_helper.rb | 4 ++-- 16 files changed, 23 insertions(+), 59 deletions(-) delete mode 100755 build/new-jenkins/docker-compose-rspec-parallel.sh delete mode 100755 build/new-jenkins/docker-compose-setup-cassandra-keyspaces.sh diff --git a/Jenkinsfile.contract-tests b/Jenkinsfile.contract-tests index 9b1f61a8abe..8d74697d0b6 100644 --- a/Jenkinsfile.contract-tests +++ b/Jenkinsfile.contract-tests @@ -52,7 +52,6 @@ pipeline { environment { BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN() COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-contract-tests.yml' - DOCKER_PROCESSES = '1' POSTGRES_PASSWORD = 'sekret' PACT_BROKER = credentials('PACT_BROKER') PACT_BROKER_USERNAME="${env.PACT_BROKER_USR}" diff --git a/Jenkinsfile.selenium.flakey_spec_catcher b/Jenkinsfile.selenium.flakey_spec_catcher index 29048d4566f..a151299997e 100644 --- a/Jenkinsfile.selenium.flakey_spec_catcher +++ b/Jenkinsfile.selenium.flakey_spec_catcher @@ -92,7 +92,7 @@ def computeTestCount() { } sh """ - docker-compose --project-name canvas-lms0 run --rm -v \$(pwd)/\$LOCAL_WORKDIR/tmp:\$DOCKER_WORKDIR/tmp \ + docker-compose run --rm -v \$(pwd)/\$LOCAL_WORKDIR/tmp:\$DOCKER_WORKDIR/tmp \ canvas \ bash -c "bundle exec rspec --dry-run ${changed_tests_arr.join(' ')} --require ./build/new-jenkins/rspec_location.rb --format RSpecLocationFormatter --out ${env.DOCKER_WORKDIR}/tmp/test_list_split" """ @@ -204,7 +204,6 @@ pipeline { // repeats each test FSC_REPEAT_FACTOR times. // Assumption: non-selenium tests take 500ms / test // Assumption: selenium tests take 500ms * SELENIUM_RATIO / test - DOCKER_PROCESSES = 1 FSC_REPEAT_FACTOR = 10 MAX_NODES = configuration.getString('fsc-max-nodes', '10') RSPEC_PROCESSES = 3 diff --git a/build/new-jenkins/docker-compose-build-up.sh b/build/new-jenkins/docker-compose-build-up.sh index 913d4435098..81e6860ca11 100755 --- a/build/new-jenkins/docker-compose-build-up.sh +++ b/build/new-jenkins/docker-compose-build-up.sh @@ -1,17 +1,12 @@ #!/bin/bash set -o errexit -o errtrace -o nounset -o pipefail -o xtrace -# ':' is a bash "no-op" and then we pass an empty argument which isn't used -parallel --will-cite ::: : - -PROCESSES=$((${DOCKER_PROCESSES:=1}-1)) docker-compose build - -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} up -d" +docker-compose up -d for service in cassandra:9160 dynamodb:8000 redis:6379; do - seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas ./build/new-jenkins/wait-for-it ${service}" + docker-compose exec -T canvas ./build/new-jenkins/wait-for-it ${service} done -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T postgres /bin/bash -c /wait-for-it" +docker-compose exec -T postgres /bin/bash -c /wait-for-it diff --git a/build/new-jenkins/docker-compose-rspec-parallel.sh b/build/new-jenkins/docker-compose-rspec-parallel.sh deleted file mode 100755 index 42a16bef8cc..00000000000 --- a/build/new-jenkins/docker-compose-rspec-parallel.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -o errexit -o errtrace -o pipefail -o xtrace -# ':' is a bash "no-op" and then we pass an empty argument which isn't used -parallel --will-cite ::: : - -# Run each group of tests in separate docker container -seq 0 $((DOCKER_PROCESSES-1)) | parallel -u "docker-compose --project-name canvas-lms{} exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM canvas bash -c 'build/new-jenkins/rspec-with-retries.sh'" diff --git a/build/new-jenkins/docker-compose-setup-cassandra-keyspaces.sh b/build/new-jenkins/docker-compose-setup-cassandra-keyspaces.sh deleted file mode 100755 index 91d794713a8..00000000000 --- a/build/new-jenkins/docker-compose-setup-cassandra-keyspaces.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -o errexit -o errtrace -o nounset -o pipefail -o xtrace -# ':' is a bash "no-op" and then we pass an empty argument which isn't used -parallel --will-cite ::: : - -PROCESSES=$((${DOCKER_PROCESSES:=1}-1)) - -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas ./build/new-jenkins/wait-for-it cassandra:9160" -create_cmd="" -for keyspace in auditors global_lookups page_views; do - create_cmd+="CREATE KEYSPACE IF NOT EXISTS ${keyspace} WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };" -done -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T cassandra cqlsh -e \"${create_cmd[@]}\"" diff --git a/build/new-jenkins/docker-compose-setup-databases.sh b/build/new-jenkins/docker-compose-setup-databases.sh index ee009106ddd..525afd7f575 100755 --- a/build/new-jenkins/docker-compose-setup-databases.sh +++ b/build/new-jenkins/docker-compose-setup-databases.sh @@ -4,7 +4,6 @@ set -o errexit -o errtrace -o nounset -o pipefail -o xtrace # ':' is a bash "no-op" and then we pass an empty argument which isn't used parallel --will-cite ::: : -PROCESSES=$((${DOCKER_PROCESSES:=1}-1)) DATABASE_PROCESSES=$((${RSPEC_PROCESSES:=1}-1)) create_cmd="" @@ -12,17 +11,14 @@ for keyspace in auditors global_lookups page_views; do create_cmd+="CREATE KEYSPACE IF NOT EXISTS ${keyspace} WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };" done -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T cassandra cqlsh -e \"${create_cmd[@]}\"" +docker-compose exec -T cassandra cqlsh -e "${create_cmd[@]}" -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas bin/rails db:migrate >> ./migrate-{}.log" -seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas bin/rails runner \"require 'switchman/test_helper'; Switchman::TestHelper.recreate_persistent_test_shards\"" +docker-compose exec -T canvas bin/rails db:migrate >> ./migrate.log +docker-compose exec -T canvas bin/rails runner "require 'switchman/test_helper'; Switchman::TestHelper.recreate_persistent_test_shards" -for i in $(seq 0 $PROCESSES); do - for keyspace in auditors global_lookups page_views; do - seq 0 $DATABASE_PROCESSES | parallel "docker-compose --project-name canvas-lms${i} exec -T cassandra bash -c 'cqlsh -e \"DESCRIBE KEYSPACE ${keyspace}\" | sed \"s/CREATE KEYSPACE ${keyspace}/CREATE KEYSPACE ${keyspace}{}/g ; s/CREATE TABLE ${keyspace}/CREATE TABLE ${keyspace}{}/g\" > ${keyspace}{} && cqlsh -f \"${keyspace}{}\"'" - done +for keyspace in auditors global_lookups page_views; do + seq 0 $DATABASE_PROCESSES | parallel "docker-compose exec -T cassandra bash -c 'cqlsh -e \"DESCRIBE KEYSPACE ${keyspace}\" | sed \"s/CREATE KEYSPACE ${keyspace}/CREATE KEYSPACE ${keyspace}{}/g ; s/CREATE TABLE ${keyspace}/CREATE TABLE ${keyspace}{}/g\" > ${keyspace}{} && cqlsh -f \"${keyspace}{}\"'" done -for i in $(seq 0 $PROCESSES); do - seq 0 $DATABASE_PROCESSES | parallel "docker-compose --project-name canvas-lms${i} exec -T postgres sh -c 'createdb -U postgres -T canvas_test canvas_test_{}'" -done +seq 0 $DATABASE_PROCESSES | parallel "docker-compose exec -T postgres sh -c 'createdb -U postgres -T canvas_test canvas_test_{}'" + diff --git a/build/new-jenkins/library/vars/rspec.groovy b/build/new-jenkins/library/vars/rspec.groovy index 2c529613b38..484e8ad1729 100644 --- a/build/new-jenkins/library/vars/rspec.groovy +++ b/build/new-jenkins/library/vars/rspec.groovy @@ -37,7 +37,6 @@ def runSeleniumSuite(total, index) { config.reruns_retry, '^./(spec|gems/plugins/.*/spec_canvas)/selenium', '.*/performance', - '1', '3', config.force_failure, config.patchsetTag @@ -65,7 +64,6 @@ def runRSpecSuite(total, index) { config.reruns_retry, '^./(spec|gems/plugins/.*/spec_canvas)/', '.*/(selenium|contracts)', - '1', '4', config.force_failure, config.patchsetTag @@ -81,7 +79,6 @@ def _runRspecTestSuite( rerunsRetry, testFilePattern, excludeRegex, - dockerProcesses, rspecProcesses, forceFailure, patchsetTag @@ -94,7 +91,6 @@ def _runRspecTestSuite( "TEST_PATTERN=$testFilePattern", "EXCLUDE_TESTS=$excludeRegex", "CI_NODE_TOTAL=$total", - "DOCKER_PROCESSES=$dockerProcesses", "RSPEC_PROCESSES=$rspecProcesses", "FORCE_FAILURE=$forceFailure", 'POSTGRES_PASSWORD=sekret', @@ -109,7 +105,7 @@ def _runRspecTestSuite( sh(script: 'build/new-jenkins/docker-compose-pull.sh', label: 'Pull Images') } sh(script: 'build/new-jenkins/docker-compose-build-up.sh', label: 'Start Containers') - sh(script: 'build/new-jenkins/docker-compose-rspec-parallel.sh', label: 'Run Tests') + sh(script: 'docker-compose exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM canvas bash -c \'build/new-jenkins/rspec-with-retries.sh\'', label: 'Run Tests') } } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) { if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { diff --git a/build/new-jenkins/rspec-flakey-spec-catcher.sh b/build/new-jenkins/rspec-flakey-spec-catcher.sh index 71807de1af2..cc8fb5e3c46 100755 --- a/build/new-jenkins/rspec-flakey-spec-catcher.sh +++ b/build/new-jenkins/rspec-flakey-spec-catcher.sh @@ -2,4 +2,4 @@ set -o errexit -o errtrace -o nounset -o pipefail -o xtrace -docker-compose --project-name "canvas-lms0" exec -T -e RSPEC_PROCESSES=$RSPEC_PROCESSES -e FORCE_FAILURE=$FORCE_FAILURE -e FSC_REPEAT_FACTOR=$FSC_REPEAT_FACTOR -e FSC_TESTS=$FSC_TESTS canvas bash -c 'build/new-jenkins/rspec-flakey-spec-catcher-parallel.sh' +docker-compose exec -T -e RSPEC_PROCESSES=$RSPEC_PROCESSES -e FORCE_FAILURE=$FORCE_FAILURE -e FSC_REPEAT_FACTOR=$FSC_REPEAT_FACTOR -e FSC_TESTS=$FSC_TESTS canvas bash -c 'build/new-jenkins/rspec-flakey-spec-catcher-parallel.sh' diff --git a/build/new-jenkins/rspec-tests.sh b/build/new-jenkins/rspec-tests.sh index 0ea591b7745..d535293fc87 100755 --- a/build/new-jenkins/rspec-tests.sh +++ b/build/new-jenkins/rspec-tests.sh @@ -7,7 +7,7 @@ parallel_index=$1 only_failures=${2-} # calculate which group to run -max=$((CI_NODE_TOTAL * (DOCKER_PROCESSES * RSPEC_PROCESSES))) +max=$((CI_NODE_TOTAL * RSPEC_PROCESSES)) group=$(((max-CI_NODE_TOTAL * $parallel_index) - CI_NODE_INDEX)) maybeOnlyFailures=() if [ "${only_failures}" = 'only-failures' ] && [ ! "${RSPEC_LOG:-}" == "1" ]; then diff --git a/build/new-jenkins/rspec-with-retries.sh b/build/new-jenkins/rspec-with-retries.sh index 73c6c665bf5..eecedce88c5 100755 --- a/build/new-jenkins/rspec-with-retries.sh +++ b/build/new-jenkins/rspec-with-retries.sh @@ -19,11 +19,11 @@ while true; do if [[ $reruns_started ]]; then if [ $1 ] && [ $1 = 'performance' ]; then - commands+=("docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --only-failures --failure-exit-code 99") + commands+=("docker-compose exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --only-failures --failure-exit-code 99") fi else if [ $1 ] && [ $1 = 'performance' ]; then - commands+=("docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --failure-exit-code 99") + commands+=("docker-compose exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --failure-exit-code 99") else for i in $(seq 0 $PROCESSES); do commands+=("build/new-jenkins/rspec-tests.sh $i") diff --git a/build/new-jenkins/test-engines.sh b/build/new-jenkins/test-engines.sh index 8b93bacd269..7508b4a3b8b 100755 --- a/build/new-jenkins/test-engines.sh +++ b/build/new-jenkins/test-engines.sh @@ -2,4 +2,4 @@ set -x -o errexit -o errtrace -o nounset -o pipefail -docker-compose --project-name canvas-lms0 exec -T canvas ./engines/test_all_engines.sh +docker-compose exec -T canvas ./engines/test_all_engines.sh diff --git a/build/new-jenkins/test-gems.sh b/build/new-jenkins/test-gems.sh index 3876a5de85a..75eec9afeac 100755 --- a/build/new-jenkins/test-gems.sh +++ b/build/new-jenkins/test-gems.sh @@ -2,4 +2,4 @@ set -x -o errexit -o errtrace -o nounset -o pipefail -docker-compose --project-name canvas-lms0 exec -T canvas ./gems/test_all_gems.sh +docker-compose exec -T canvas ./gems/test_all_gems.sh diff --git a/build/new-jenkins/xbrowser-test.sh b/build/new-jenkins/xbrowser-test.sh index 0f6b303e18a..da6c1cc9d01 100755 --- a/build/new-jenkins/xbrowser-test.sh +++ b/build/new-jenkins/xbrowser-test.sh @@ -2,4 +2,4 @@ set -x -o errexit -o errtrace -o nounset -o pipefail -docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts --tag xbrowser spec/selenium/ +docker-compose exec -T canvas bundle exec rspec --options spec/spec.opts --tag xbrowser spec/selenium/ diff --git a/docker-compose.new-jenkins.yml b/docker-compose.new-jenkins.yml index c3c8dd9c56c..bd11cb40247 100644 --- a/docker-compose.new-jenkins.yml +++ b/docker-compose.new-jenkins.yml @@ -23,7 +23,7 @@ services: # parallel_tests CI_NODE_TOTAL: CI_NODE_INDEX: - DOCKER_PROCESSES: + RSPEC_PROCESSES: EXCLUDE_TESTS: TEST_PATTERN: # rspec-parallel-log build diff --git a/spec/coverage_tool.rb b/spec/coverage_tool.rb index f3057a13b15..18e75e30e55 100644 --- a/spec/coverage_tool.rb +++ b/spec/coverage_tool.rb @@ -28,7 +28,7 @@ class CoverageTool # no formatting by default, just get the json SimpleCov.at_exit { # generate an HTML report if this is running locally / not on jenkins: - SimpleCov.result.format! unless ENV['DOCKER_PROCESSES'] + SimpleCov.result.format! unless ENV['RSPEC_PROCESSES'] SimpleCov.result } end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 57794972d65..f31bb7629e6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -354,8 +354,8 @@ RSpec.configure do |config| config.include PGCollkeyHelper config.project_source_dirs << "gems" # so that failures here are reported properly - # DOCKER_PROCESSES is only used on Jenkins and we only care to have RspecJunitFormatter on Jenkins. - if ENV['DOCKER_PROCESSES'] + # RSPEC_PROCESSES is only used on Jenkins and we only care to have RspecJunitFormatter on Jenkins. + if ENV['RSPEC_PROCESSES'] file = "log/results/results-#{ENV.fetch('PARALLEL_INDEX', '0').to_i}.xml" # if file already exists this is a rerun of a failed spec, don't generate new xml. config.add_formatter "RspecJunitFormatter", file unless File.file?(file)