refactor FORCE_FAILURE logic for clarity

refs DE-285

Test Plan:
1. FSC works for main build
2. FSC works for plugin build
3. FSC works for force-failure path

Change-Id: If1c632fa047b6a7776f458baf5aa3926e62c5070
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/248421
QA-Review: Aaron Ogata <aogata@instructure.com>
Product-Review: Aaron Ogata <aogata@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Ryan Norton <rnorton@instructure.com>

Jenkinsfile

@@ -490,7 +490,13 @@ pipeline {
                 echo 'no migrations added, skipping CDC Schema check'
               }
 
-              if (!configuration.isChangeMerged() && dir(env.LOCAL_WORKDIR){ (sh(script: '${WORKSPACE}/build/new-jenkins/spec-changes.sh', returnStatus: true) == 0) }) {
+              if (
+                !configuration.isChangeMerged() &&
+                (
+                  dir(env.LOCAL_WORKDIR){ (sh(script: '${WORKSPACE}/build/new-jenkins/spec-changes.sh', returnStatus: true) == 0) } ||
+                  configuration.forceFailureFSC() == '1'
+                )
+              ) {
                 echo 'adding Flakey Spec Catcher'
                 stages['Flakey Spec Catcher'] = {
                   skipIfPreviouslySuccessful("flakey-spec-catcher") {

Jenkinsfile.selenium.flakey_spec_catcher

@@ -67,6 +67,10 @@ def computeTestCount() {
   sh 'rm -vrf $LOCAL_WORKDIR/tmp'
   sh 'mkdir -v $LOCAL_WORKDIR/tmp'
   sh 'chmod -vv 777 $LOCAL_WORKDIR/tmp'
+
+  if (FORCE_FAILURE == '1') {
+    changed_tests = 'spec/force_failure_spec.rb'
+  } else {
     sh '''
       docker run --volume $(pwd)/$LOCAL_WORKDIR/.git:$DOCKER_WORKDIR/.git \
                  --volume $(pwd)/$LOCAL_WORKDIR/tmp:$DOCKER_WORKDIR/tmp \
@@ -77,6 +81,8 @@ def computeTestCount() {
     '''
 
     changed_tests = readFile("$env.LOCAL_WORKDIR/tmp/test_list").trim()
+  }
+
   echo "raw result from catcher: \n====\n$changed_tests\n===="
 
   def changed_tests_arr = changed_tests.split('\n')
@@ -186,25 +192,19 @@ pipeline {
               setupNode()
             }
 
-            if(FORCE_FAILURE != '1') {
             stage("Compute Build Distribution") {
               computeTestCount()
             }
-            }
 
-            if(partitions.size() > 0 || FORCE_FAILURE == '1') {
             stage("Run Flakey Spec Catcher") {
-                if (FORCE_FAILURE == '1') {
+              if(partitions.size() == 0) {
-                  echo "running force failure"
+                return
-                  executeFlakeySpecCatcher()
+              } else if (partitions.size() == 1) {
-                }
-                else if (partitions.size() <= 1) {
                 echo "running on this node"
                 withEnv(["FSC_TESTS=${partitions[0].join(',')}"]) {
                   executeFlakeySpecCatcher()
                 }
-                }
+              } else {
-                else {
                 echo "running on multiple nodes: ${partitions.size()}"
                 def nodes = [:]
                 for(int i = 0; i < partitions.size(); i++) {
@@ -236,5 +236,4 @@ pipeline {
       }
     }
   }
-  }
 }

build/new-jenkins/rspec-flakey-spec-catcher.sh

@@ -2,13 +2,8 @@
 
 set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
 
-if [ "${FORCE_FAILURE:-}" == '1' ]; then
+if [ "${IS_PLUGIN}" == "1" ]; then
-  docker-compose --project-name canvas-lms0 exec -T -e FORCE_FAILURE=1 canvas bundle exec flakey_spec_catcher \
+   docker-compose --project-name canvas-lms0 exec -T -e FORCE_FAILURE=$FORCE_FAILURE canvas \
-      --repeat=$FSC_REPEAT_FACTOR \
-      --output=/usr/src/app/tmp/fsc.out \
-      --test=spec/force_failure_spec.rb
-elif [ "${IS_PLUGIN}" == "1" ]; then
-   docker-compose --project-name canvas-lms0 exec -T canvas \
          bash -c "cd $DOCKER_WORKDIR && BUNDLE_GEMFILE=../../../Gemfile bundle exec flakey_spec_catcher \
          --repeat=$FSC_REPEAT_FACTOR \
          --output=/usr/src/app/tmp/fsc.out \
@@ -16,7 +11,7 @@ elif [ "${IS_PLUGIN}" == "1" ]; then
          --test=$FSC_TESTS \
          --rspec-options '-I spec_canvas'"
 else
-  docker-compose --project-name canvas-lms0 exec -T canvas \
+  docker-compose --project-name canvas-lms0 exec -T -e FORCE_FAILURE=$FORCE_FAILURE canvas \
       bash -c "cd $DOCKER_WORKDIR && bundle exec flakey_spec_catcher \
       --repeat=$FSC_REPEAT_FACTOR \
       --output=/usr/src/app/tmp/fsc.out \