spec: generate html report

closes: CORE-3234

Change-Id: I8a85ed899e1097e5c91276d97c8339931cda2e1a
Reviewed-on: https://gerrit.instructure.com/208309
Tested-by: Jenkins
Reviewed-by: Clay Diffrient <cdiffrient@instructure.com>
QA-Review: James Butters <jbutters@instructure.com>
Product-Review: James Butters <jbutters@instructure.com>

Jenkinsfile.rspec

@@ -44,7 +44,29 @@ pipeline {
       }
 
       post {
+        unsuccessful {
+          // copy spec failures to local
+          sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp'
+          script {
+            def htmlFiles
+            // find all results files
+            dir ('tmp') {
+              htmlFiles = findFiles glob: '**/index.html'
+            }
+            // publish html
+            publishHTML target: [
+              allowMissing: false,
+              alwaysLinkToLastBuild: false,
+              keepAll: true,
+              reportDir: "tmp",
+              reportFiles: htmlFiles.join(','),
+              reportName: 'Test Failures'
+              ]
+          }
+        }
+
         cleanup {
+          sh 'rm -rf ./tmp/spec_failures'
           sh 'build/new-jenkins/docker-cleanup.sh'
         }
       }

Jenkinsfile.selenium.chrome

@@ -46,7 +46,29 @@ pipeline {
             }
           }
           post {
+            unsuccessful {
+              // copy spec failures to local
+              sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp'
+              script {
+                def htmlFiles
+                // find all results files
+                dir ('tmp') {
+                  htmlFiles = findFiles glob: '**/index.html'
+                }
+                // publish html
+                publishHTML target: [
+                  allowMissing: false,
+                  alwaysLinkToLastBuild: false,
+                  keepAll: true,
+                  reportDir: "tmp",
+                  reportFiles: htmlFiles.join(','),
+                  reportName: 'Test Failures'
+                  ]
+              }
+            }
+
             cleanup {
+              sh 'rm -rf ./tmp/spec_failures'
               sh 'build/new-jenkins/docker-cleanup.sh'
             }
           }

Jenkinsfile.selenium.performance.chrome

@@ -45,7 +45,29 @@ pipeline {
       }
 
       post {
+        unsuccessful {
+          // copy spec failures to local
+          sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp'
+          script {
+            def htmlFiles
+            // find all results files
+            dir ('tmp') {
+              htmlFiles = findFiles glob: '**/index.html'
+            }
+            // publish html
+            publishHTML target: [
+              allowMissing: false,
+              alwaysLinkToLastBuild: false,
+              keepAll: true,
+              reportDir: "tmp",
+              reportFiles: htmlFiles.join(','),
+              reportName: 'Test Failures'
+              ]
+          }
+        }
+
         cleanup {
+          sh 'rm -rf ./tmp/spec_failures'
           sh 'build/new-jenkins/docker-cleanup.sh'
         }
       }

Jenkinsfile.xbrowser

@@ -44,7 +44,29 @@ pipeline {
       }
 
       post {
+        unsuccessful {
+          // copy spec failures to local
+          sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp'
+          script {
+            def htmlFiles
+            // find all results files
+            dir ('tmp') {
+              htmlFiles = findFiles glob: '**/index.html'
+            }
+            // publish html
+            publishHTML target: [
+              allowMissing: false,
+              alwaysLinkToLastBuild: false,
+              keepAll: true,
+              reportDir: "tmp",
+              reportFiles: htmlFiles.join(','),
+              reportName: 'Test Failures'
+            ]
+          }
+        }
+
         cleanup {
+          sh 'rm -rf ./tmp/spec_failures'
           sh 'build/new-jenkins/docker-cleanup.sh'
         }
       }

build/new-jenkins/rspec-tests.sh

@@ -4,4 +4,5 @@
 # spec_files=`find spec {gems,vendor}/plugins/*/spec_canvas -type f -name '*_spec.rb'|grep -v '/selenium/'|tr '\n' ' '`
 
 # Todo: run the specs in spec_files
-docker-compose exec -T web bundle exec rspec -f doc --format html --out results.html spec/lib/*_spec.rb
+# -O spec/spec.opts runs rspec with our formatters for failure reports
+docker-compose exec -T web bundle exec rspec -O spec/spec.opts spec/lib/*_spec.rb

build/new-jenkins/smoke-test.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker-compose exec -T web bundle exec rspec spec/selenium/login_logout_spec.rb -f doc --format html --out results.html
+# -O spec/spec.opts runs rspec with our formatters for failure reports
+docker-compose exec -T web bundle exec rspec -O spec/spec.opts spec/selenium/login_logout_spec.rb

build/new-jenkins/xbrowser-test.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker-compose exec -T web bundle exec rspec -f doc --format html --out results.html --tag xbrowser spec/selenium/
+# -O spec/spec.opts runs rspec with our formatters for failure reports
+docker-compose exec -T web bundle exec rspec -O spec/spec.opts --tag xbrowser spec/selenium/

spec/spec.opts

@@ -1,4 +1,6 @@
---colour
---require ./spec/formatters/nested_instafail_formatter.rb
---format RSpec::NestedInstafailFormatter
---format html:tmp/spec_html/index.html
+--format doc
+--require './spec/formatters/error_context/html_page_formatter.rb'
+--format ErrorContext::HTMLPageFormatter
+--tty
+--color
+--profile 50