spec: knapsack for selenium

Change-Id: I30c0c0006de8779dea1743d6c626022756f05d5a
Reviewed-on: https://gerrit.instructure.com/210564
Tested-by: Jenkins
Reviewed-by: James Williams <jamesw@instructure.com>
Reviewed-by: Rob Orton <rob@instructure.com>
QA-Review: James Butters <jbutters@instructure.com>
Product-Review: James Butters <jbutters@instructure.com>

Jenkinsfile.rspec

@@ -56,7 +56,7 @@ pipeline {
                         sh 'build/new-jenkins/docker-compose-create-migrate-database.sh'
                         def retries = 1
                         result = sh (
-                        script: 'build/new-jenkins/rspec-tests.sh',
+                          script: 'build/new-jenkins/rspec-tests.sh',
                           returnStatus:true
                         )
                         while (result != 0 && retries > 0) {

Jenkinsfile.selenium.chrome

@@ -18,6 +18,7 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+def ci_node_total = 20; // how many nodes to run on
 pipeline {
   agent none
   options {
@@ -29,49 +30,106 @@ pipeline {
     // 'refs/changes/63/181863/8' -> '63.181863.8'
     NAME = "${env.GERRIT_REFSPEC}".minus('refs/changes/').replaceAll('/','.')
     PATCHSET_TAG = "$DOCKER_REGISTRY_FQDN/jenkins/canvas-lms:$NAME"
+    KNAPSACK_ENABLED = 1
+    KNAPSACK_GENERATE_REPORT = 'false'
+    KNAPSACK_TEST_FILE_PATTERN = '{spec/selenium,gems/plugins/*/spec_canvas/selenium}/**/*_spec.rb'
+    KNAPSACK_EXCLUDE_REGEX = '/performance/'
+    KNAPSACK_TEST_DIR = 'spec'
   }
 
   stages {
-    stage ('Run Tests Parallel') {
-      parallel {
-        stage('Selenium Tests') {
-          agent { label 'canvas-docker' }
-          steps {
-            timeout(time: 60) {
-              sh 'printenv | sort'
-              sh 'build/new-jenkins/docker-compose-build-up.sh'
-              sh 'build/new-jenkins/docker-compose-create-migrate-database.sh'
-              // Todo: create script for selenium tests
-              sh 'build/new-jenkins/smoke-test.sh'
-            }
-          }
-          post {
-            unsuccessful {
-              // copy spec failures to local
-              sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp'
-              script {
-                def htmlFiles
-                // find all results files
-                dir ('tmp') {
-                  htmlFiles = findFiles glob: '**/index.html'
+    stage ('Distribute Selenium Tests') {
+      steps {
+        script {
+          def nodes = [:];
+          for(int i = 0; i < ci_node_total; i++) {
+            def index = i;
+            nodes["selenium set ${(i+1).toString().padLeft(2, '0')}"] = {
+              withEnv(["CI_NODE_INDEX=$index", "CI_NODE_TOTAL=$ci_node_total"]) {
+                node('canvas-docker') {
+                  stage("Running Selenium Set ${index}") {
+                    try {
+                      checkout scm
+                      sh 'rm -rf ./tmp/spec_failures'
+                      timeout(time: 60) {
+                        sh 'printenv | sort'
+                        sh 'build/new-jenkins/docker-compose-build-up.sh'
+                        sh 'build/new-jenkins/docker-compose-create-migrate-database.sh'
+                        def retries = 1
+                        result = sh (
+                          script: 'build/new-jenkins/rspec-tests.sh',
+                          returnStatus:true
+                        )
+                        while (result != 0 && retries > 0) {
+                          println "Re-Running failed tests"
+                          result = sh (
+                            script: 'build/new-jenkins/rspec-tests.sh only-failures',
+                            returnStatus: true
+                          )
+                          retries--
+                        }
+                        if (result != 0) {
+                          error("Rspec exited exit code ${result}")
+                        }
+                      }
+                    }
+                    catch (ex) {
+                      // copy spec failures to local
+                      sh 'mkdir -p tmp'
+                      sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp/spec_failures/'
+                      throw ex
+                    }
+                    finally {
+                      dir ('tmp') {
+                        stash name: "selenium_failures_${index}", includes: 'spec_failures/**/*', allowEmpty: true
+                      }
+                      sh 'rm -rf ./tmp/spec_failures'
+                      sh 'build/new-jenkins/docker-cleanup.sh'
+                    }
+                  }
                 }
-                // publish html
-                publishHTML target: [
-                  allowMissing: false,
-                  alwaysLinkToLastBuild: false,
-                  keepAll: true,
-                  reportDir: "tmp",
-                  reportFiles: htmlFiles.join(','),
-                  reportName: 'Test Failures'
-                  ]
               }
             }
-
-            cleanup {
-              sh 'rm -rf ./tmp/spec_failures'
-              sh 'build/new-jenkins/docker-cleanup.sh'
-            }
           }
+          parallel(nodes);
+        }
+      }
+    }
+  }
+
+  post {
+    failure {
+      script {
+        node {
+          sh 'rm -rf ./compiled_failures'
+          def htmlFiles;
+          dir('compiled_failures') {
+            for(int i = 0; i < ci_node_total; i++) {
+              def index = i;
+              dir ("node_${index}") {
+                unstash "selenium_failures_${index}"
+              }
+            }
+            htmlFiles = findFiles glob: '**/index.html'
+
+            def indexHtml = "<body style=\"font-family:sans-serif;line-height:1.25;font-size:14px\">"
+            htmlFiles.each {
+              def spec = (it =~ /.*spec_failures\/(.*)\/index/)[0][1]
+              indexHtml += "<a href=\"${it}\">${spec}</a><br>"
+            }
+            indexHtml += "</body>"
+            writeFile file: "index.html", text: indexHtml
+          }
+
+          publishHTML target: [
+            allowMissing: false,
+            alwaysLinkToLastBuild: false,
+            keepAll: true,
+            reportDir: 'compiled_failures',
+            reportFiles: "index.html," + htmlFiles.join(','),
+            reportName: 'Test Failures'
+          ]
+          sh 'rm -rf ./compiled_failures'
         }
       }
     }

build/Dockerfile.template

@@ -32,6 +32,7 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
 <% unless production? -%>
        postgresql-client-9.5 \
        unzip \
+       pbzip2 \
        fontforge \
 <% end -%>
   && apt-get clean \

build/new-jenkins/rspec-tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ $1 ] && [ $1 = 'only-failures' ]; then
-  docker-compose exec -T web bundle exec rake 'knapsack:rspec[-O spec/spec.opts --only-failures]'
+  docker-compose exec -T web bundle exec rspec -O spec/spec.opts --only-failures
 else
   docker-compose exec -T web bundle exec rake 'knapsack:rspec[-O spec/spec.opts]'
 fi

docker-compose.new-jenkins-selenium.yml

@@ -1,12 +1,29 @@
 # it is intended that this be used exclusive of all other docker-compose.*yml files in CI
-version: "2.1"
+version: "2.3"
 services:
   web:
     links:
       - selenium-chrome
+      - canvasrceapi
     environment:
       remote_url: http://selenium-chrome:4444/wd/hub
       browser: chrome
+      RCE_HOST: "http://canvasrceapi"
+      USE_OPTIMIZED_JS: 'true'
+      SASS_STYLE: 'compressed'
 
   selenium-chrome:
     build: ./docker-compose/selenium-chrome
+
+  canvasrceapi:
+    image: starlord.inscloudgate.net/jeremyp/canvas-rce-api_web
+    environment:
+      ECOSYSTEM_KEY: "astringthatisactually32byteslong"
+      ECOSYSTEM_SECRET: "astringthatisactually32byteslong"
+      HTTP_PROTOCOL_OVERRIDE: "http"
+      NODE_ENV: production
+      PASSENGER_MIN_INSTANCES: 2
+      PASSENGER_MAX_POOL_SIZE: 6
+      NGINX_WORKER_CONNECTIONS: 2048
+    ports:
+      - "3001:80"

docker-compose.new-jenkins.yml

@@ -1,5 +1,5 @@
 # it is intended that this be used exclusive of all other docker-compose.*yml files in CI
-version: "2.1"
+version: "2.3"
 services:
   web:
     # use master if NAME is unavailable

docker-compose/selenium-chrome/Dockerfile

@@ -1,5 +1,5 @@
 # Keep this image version tag synced with Gemfile.d/test.rb
-FROM selenium/standalone-chrome-debug:3.141.59-oxygen
+FROM selenium/standalone-chrome-debug:3.141.59-uranium
 
 COPY entry_point.sh /opt/bin/custom_entry_point.sh
 USER root

spec/factories/account_factory.rb

@@ -25,7 +25,7 @@ module Factories
     allow(Canvas::DynamicSettings).to receive(:find).with(any_args).and_call_original
     allow(Canvas::DynamicSettings).to receive(:find).with("rich-content-service", default_ttl: 5.minutes).and_return(
       ActiveSupport::HashWithIndifferentAccess.new({
-        "app-host":"http://localhost:3001",
+        "app-host":ENV['RCE_HOST'] || "http://localhost:3001",
       })
     )
     allow(Canvas::DynamicSettings).to receive(:find).with("canvas").and_return(

spec/selenium/test_setup/selenium_driver_setup.rb

@@ -366,15 +366,22 @@ module SeleniumDriverSetup
     end
 
     def desired_capabilities
-      caps = Selenium::WebDriver::Remote::Capabilities.send(browser)
-      caps.version = CONFIG[:version] unless CONFIG[:version].nil?
-      caps.platform = CONFIG[:platform] unless CONFIG[:platform].nil?
-      caps['name'] = "#{CONFIG[:platform]} - #{CONFIG[:browser]}-#{CONFIG[:version]}" unless CONFIG[:platform].nil?
-      caps["tunnel-identifier"] = CONFIG[:tunnel_id] unless CONFIG[:tunnel_id].nil?
-      caps['selenium-version'] = "3.4.0"
-      caps[:unexpectedAlertBehaviour] = 'ignore'
-      caps[:elementScrollBehavior] = 1
-      caps['screen-resolution'] = "1280x1024"
+      case browser
+      when :firefox
+        # TODO: options for firefox driver
+      when :chrome
+        caps = Selenium::WebDriver::Remote::Capabilities.chrome
+        caps['chromeOptions'] = {
+          args: %w[disable-dev-shm-usage no-sandbox],
+          w3c: false
+        }
+      when :edge
+        # TODO: options for edge driver
+      when :safari
+        # TODO: options for safari driver
+      else
+        raise "unsupported browser #{browser}"
+      end
       caps
     end
 

spec/spec.opts

@@ -1,6 +1,12 @@
 --format doc
+--require './spec/formatters/rerun_formatter.rb'
+--format RSpec::RerunFormatter
+--require './spec/formatters/abort_on_consistent_badness_formatter.rb'
+--require './spec/formatters/error_context/stderr_formatter.rb'
 --require './spec/formatters/error_context/html_page_formatter.rb'
+--format AbortOnConsistentBadnessFormatter
 --format ErrorContext::HTMLPageFormatter
+--format ErrorContext::StderrFormatter
 --tty
 --color
 --profile 50