From 56a487811be69a2ad7f11989e53567b577fe20c0 Mon Sep 17 00:00:00 2001 From: James Williams Date: Wed, 11 Sep 2019 09:49:35 -0600 Subject: [PATCH] spec: knapsack for selenium Change-Id: I30c0c0006de8779dea1743d6c626022756f05d5a Reviewed-on: https://gerrit.instructure.com/210564 Tested-by: Jenkins Reviewed-by: James Williams Reviewed-by: Rob Orton QA-Review: James Butters Product-Review: James Butters --- Jenkinsfile.rspec | 2 +- Jenkinsfile.selenium.chrome | 130 +++++++++++++----- build/Dockerfile.template | 1 + build/new-jenkins/rspec-tests.sh | 2 +- docker-compose.new-jenkins-selenium.yml | 19 ++- docker-compose.new-jenkins.yml | 2 +- docker-compose/selenium-chrome/Dockerfile | 2 +- spec/factories/account_factory.rb | 2 +- .../test_setup/selenium_driver_setup.rb | 25 ++-- spec/spec.opts | 6 + 10 files changed, 140 insertions(+), 51 deletions(-) diff --git a/Jenkinsfile.rspec b/Jenkinsfile.rspec index 01be9c0e498..ea1aedce198 100644 --- a/Jenkinsfile.rspec +++ b/Jenkinsfile.rspec @@ -56,7 +56,7 @@ pipeline { sh 'build/new-jenkins/docker-compose-create-migrate-database.sh' def retries = 1 result = sh ( - script: 'build/new-jenkins/rspec-tests.sh', + script: 'build/new-jenkins/rspec-tests.sh', returnStatus:true ) while (result != 0 && retries > 0) { diff --git a/Jenkinsfile.selenium.chrome b/Jenkinsfile.selenium.chrome index 94a3f78c52e..06a86829fb0 100644 --- a/Jenkinsfile.selenium.chrome +++ b/Jenkinsfile.selenium.chrome @@ -18,6 +18,7 @@ * with this program. If not, see . */ +def ci_node_total = 20; // how many nodes to run on pipeline { agent none options { @@ -29,49 +30,106 @@ pipeline { // 'refs/changes/63/181863/8' -> '63.181863.8' NAME = "${env.GERRIT_REFSPEC}".minus('refs/changes/').replaceAll('/','.') PATCHSET_TAG = "$DOCKER_REGISTRY_FQDN/jenkins/canvas-lms:$NAME" + KNAPSACK_ENABLED = 1 + KNAPSACK_GENERATE_REPORT = 'false' + KNAPSACK_TEST_FILE_PATTERN = '{spec/selenium,gems/plugins/*/spec_canvas/selenium}/**/*_spec.rb' + KNAPSACK_EXCLUDE_REGEX = '/performance/' + KNAPSACK_TEST_DIR = 'spec' } stages { - stage ('Run Tests Parallel') { - parallel { - stage('Selenium Tests') { - agent { label 'canvas-docker' } - steps { - timeout(time: 60) { - sh 'printenv | sort' - sh 'build/new-jenkins/docker-compose-build-up.sh' - sh 'build/new-jenkins/docker-compose-create-migrate-database.sh' - // Todo: create script for selenium tests - sh 'build/new-jenkins/smoke-test.sh' - } - } - post { - unsuccessful { - // copy spec failures to local - sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp' - script { - def htmlFiles - // find all results files - dir ('tmp') { - htmlFiles = findFiles glob: '**/index.html' + stage ('Distribute Selenium Tests') { + steps { + script { + def nodes = [:]; + for(int i = 0; i < ci_node_total; i++) { + def index = i; + nodes["selenium set ${(i+1).toString().padLeft(2, '0')}"] = { + withEnv(["CI_NODE_INDEX=$index", "CI_NODE_TOTAL=$ci_node_total"]) { + node('canvas-docker') { + stage("Running Selenium Set ${index}") { + try { + checkout scm + sh 'rm -rf ./tmp/spec_failures' + timeout(time: 60) { + sh 'printenv | sort' + sh 'build/new-jenkins/docker-compose-build-up.sh' + sh 'build/new-jenkins/docker-compose-create-migrate-database.sh' + def retries = 1 + result = sh ( + script: 'build/new-jenkins/rspec-tests.sh', + returnStatus:true + ) + while (result != 0 && retries > 0) { + println "Re-Running failed tests" + result = sh ( + script: 'build/new-jenkins/rspec-tests.sh only-failures', + returnStatus: true + ) + retries-- + } + if (result != 0) { + error("Rspec exited exit code ${result}") + } + } + } + catch (ex) { + // copy spec failures to local + sh 'mkdir -p tmp' + sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp/spec_failures/' + throw ex + } + finally { + dir ('tmp') { + stash name: "selenium_failures_${index}", includes: 'spec_failures/**/*', allowEmpty: true + } + sh 'rm -rf ./tmp/spec_failures' + sh 'build/new-jenkins/docker-cleanup.sh' + } + } } - // publish html - publishHTML target: [ - allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll: true, - reportDir: "tmp", - reportFiles: htmlFiles.join(','), - reportName: 'Test Failures' - ] } } - - cleanup { - sh 'rm -rf ./tmp/spec_failures' - sh 'build/new-jenkins/docker-cleanup.sh' - } } + parallel(nodes); + } + } + } + } + + post { + failure { + script { + node { + sh 'rm -rf ./compiled_failures' + def htmlFiles; + dir('compiled_failures') { + for(int i = 0; i < ci_node_total; i++) { + def index = i; + dir ("node_${index}") { + unstash "selenium_failures_${index}" + } + } + htmlFiles = findFiles glob: '**/index.html' + + def indexHtml = "" + htmlFiles.each { + def spec = (it =~ /.*spec_failures\/(.*)\/index/)[0][1] + indexHtml += "${spec}
" + } + indexHtml += "" + writeFile file: "index.html", text: indexHtml + } + + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'compiled_failures', + reportFiles: "index.html," + htmlFiles.join(','), + reportName: 'Test Failures' + ] + sh 'rm -rf ./compiled_failures' } } } diff --git a/build/Dockerfile.template b/build/Dockerfile.template index 12222f0d6f0..b21a3e63915 100644 --- a/build/Dockerfile.template +++ b/build/Dockerfile.template @@ -32,6 +32,7 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \ <% unless production? -%> postgresql-client-9.5 \ unzip \ + pbzip2 \ fontforge \ <% end -%> && apt-get clean \ diff --git a/build/new-jenkins/rspec-tests.sh b/build/new-jenkins/rspec-tests.sh index 5a376365d34..d918063b804 100755 --- a/build/new-jenkins/rspec-tests.sh +++ b/build/new-jenkins/rspec-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ $1 ] && [ $1 = 'only-failures' ]; then - docker-compose exec -T web bundle exec rake 'knapsack:rspec[-O spec/spec.opts --only-failures]' + docker-compose exec -T web bundle exec rspec -O spec/spec.opts --only-failures else docker-compose exec -T web bundle exec rake 'knapsack:rspec[-O spec/spec.opts]' fi diff --git a/docker-compose.new-jenkins-selenium.yml b/docker-compose.new-jenkins-selenium.yml index de6ff25a908..ecde441a42d 100644 --- a/docker-compose.new-jenkins-selenium.yml +++ b/docker-compose.new-jenkins-selenium.yml @@ -1,12 +1,29 @@ # it is intended that this be used exclusive of all other docker-compose.*yml files in CI -version: "2.1" +version: "2.3" services: web: links: - selenium-chrome + - canvasrceapi environment: remote_url: http://selenium-chrome:4444/wd/hub browser: chrome + RCE_HOST: "http://canvasrceapi" + USE_OPTIMIZED_JS: 'true' + SASS_STYLE: 'compressed' selenium-chrome: build: ./docker-compose/selenium-chrome + + canvasrceapi: + image: starlord.inscloudgate.net/jeremyp/canvas-rce-api_web + environment: + ECOSYSTEM_KEY: "astringthatisactually32byteslong" + ECOSYSTEM_SECRET: "astringthatisactually32byteslong" + HTTP_PROTOCOL_OVERRIDE: "http" + NODE_ENV: production + PASSENGER_MIN_INSTANCES: 2 + PASSENGER_MAX_POOL_SIZE: 6 + NGINX_WORKER_CONNECTIONS: 2048 + ports: + - "3001:80" diff --git a/docker-compose.new-jenkins.yml b/docker-compose.new-jenkins.yml index a2cc2027848..87c8152bf50 100644 --- a/docker-compose.new-jenkins.yml +++ b/docker-compose.new-jenkins.yml @@ -1,5 +1,5 @@ # it is intended that this be used exclusive of all other docker-compose.*yml files in CI -version: "2.1" +version: "2.3" services: web: # use master if NAME is unavailable diff --git a/docker-compose/selenium-chrome/Dockerfile b/docker-compose/selenium-chrome/Dockerfile index 121f7b636b6..63553150ba1 100644 --- a/docker-compose/selenium-chrome/Dockerfile +++ b/docker-compose/selenium-chrome/Dockerfile @@ -1,5 +1,5 @@ # Keep this image version tag synced with Gemfile.d/test.rb -FROM selenium/standalone-chrome-debug:3.141.59-oxygen +FROM selenium/standalone-chrome-debug:3.141.59-uranium COPY entry_point.sh /opt/bin/custom_entry_point.sh USER root diff --git a/spec/factories/account_factory.rb b/spec/factories/account_factory.rb index 10ef87c6db8..82da2c8484f 100644 --- a/spec/factories/account_factory.rb +++ b/spec/factories/account_factory.rb @@ -25,7 +25,7 @@ module Factories allow(Canvas::DynamicSettings).to receive(:find).with(any_args).and_call_original allow(Canvas::DynamicSettings).to receive(:find).with("rich-content-service", default_ttl: 5.minutes).and_return( ActiveSupport::HashWithIndifferentAccess.new({ - "app-host":"http://localhost:3001", + "app-host":ENV['RCE_HOST'] || "http://localhost:3001", }) ) allow(Canvas::DynamicSettings).to receive(:find).with("canvas").and_return( diff --git a/spec/selenium/test_setup/selenium_driver_setup.rb b/spec/selenium/test_setup/selenium_driver_setup.rb index 1db83837222..da0ccb52d06 100644 --- a/spec/selenium/test_setup/selenium_driver_setup.rb +++ b/spec/selenium/test_setup/selenium_driver_setup.rb @@ -366,15 +366,22 @@ module SeleniumDriverSetup end def desired_capabilities - caps = Selenium::WebDriver::Remote::Capabilities.send(browser) - caps.version = CONFIG[:version] unless CONFIG[:version].nil? - caps.platform = CONFIG[:platform] unless CONFIG[:platform].nil? - caps['name'] = "#{CONFIG[:platform]} - #{CONFIG[:browser]}-#{CONFIG[:version]}" unless CONFIG[:platform].nil? - caps["tunnel-identifier"] = CONFIG[:tunnel_id] unless CONFIG[:tunnel_id].nil? - caps['selenium-version'] = "3.4.0" - caps[:unexpectedAlertBehaviour] = 'ignore' - caps[:elementScrollBehavior] = 1 - caps['screen-resolution'] = "1280x1024" + case browser + when :firefox + # TODO: options for firefox driver + when :chrome + caps = Selenium::WebDriver::Remote::Capabilities.chrome + caps['chromeOptions'] = { + args: %w[disable-dev-shm-usage no-sandbox], + w3c: false + } + when :edge + # TODO: options for edge driver + when :safari + # TODO: options for safari driver + else + raise "unsupported browser #{browser}" + end caps end diff --git a/spec/spec.opts b/spec/spec.opts index 32dcb92c1ff..70be60e6a17 100644 --- a/spec/spec.opts +++ b/spec/spec.opts @@ -1,6 +1,12 @@ --format doc +--require './spec/formatters/rerun_formatter.rb' +--format RSpec::RerunFormatter +--require './spec/formatters/abort_on_consistent_badness_formatter.rb' +--require './spec/formatters/error_context/stderr_formatter.rb' --require './spec/formatters/error_context/html_page_formatter.rb' +--format AbortOnConsistentBadnessFormatter --format ErrorContext::HTMLPageFormatter +--format ErrorContext::StderrFormatter --tty --color --profile 50