Add splunk node wait times and better splunk helper usage

test plan:
You can see the numbers on the builds dashboard

fixes: CCI-357

flag= none

Change-Id: Ic1efb3df107b803c65af79543d346a4f95a110cf
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/236354
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Rex Fleischer <rfleischer@instructure.com>
Reviewed-by: James Butters <jbutters@instructure.com>
QA-Review: James Butters <jbutters@instructure.com>
Product-Review: Jacob Powell <spowell@instructure.com>

Jenkinsfile

@@ -442,8 +442,7 @@ pipeline {
           rspec.uploadSeleniumFailures()
           rspec.uploadRSpecFailures()
           load('build/new-jenkins/groovy/reports.groovy').sendFailureMessageIfPresent()
-          def splunk = load 'build/new-jenkins/groovy/splunk.groovy'
-          splunk.upload([splunk.event('jenkins.build.duration', ['duration': "$currentBuild.duration"])])
+          load('build/new-jenkins/groovy/splunk.groovy').uploadEvent('jenkins.build.duration', ['duration': "$currentBuild.duration"])
         }
       }
     }

build/new-jenkins/groovy/datadog.groovy

@@ -40,8 +40,7 @@ def runDataDogForMetric(name, block) {
   finally {
     def duration = TimeCategory.minus(new Date(), timeStart).toMilliseconds()
     hackyMetricSend("jenkins.stage.elapsedTimeDist", duration, ["stage:${name}"])
-    def splunk = load 'build/new-jenkins/groovy/splunk.groovy'
-    splunk.upload([splunk.event('jenkins.stage.duration', ["duration": duration, "stage": name])])
+    load('build/new-jenkins/groovy/splunk.groovy').uploadEvent('jenkins.stage.duration', ["duration": duration, "stage": name])
   }
 }
 

build/new-jenkins/groovy/distribution.groovy

@@ -16,6 +16,8 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+import groovy.time.*
+
 def runDatadogMetric(name, extraTags, body) {
   def dd = load('build/new-jenkins/groovy/datadog.groovy')
   dd.runDataDogForMetricWithExtraTags(name,extraTags,body)
@@ -42,11 +44,14 @@ def appendStagesAsBuildNodes(nodes,
     def index = i;
     // we cant use String.format, so... yea
     def stage_name = "$stage_name_prefix ${(index + 1).toString().padLeft(2, '0')}"
+    def timeStart = new Date()
     nodes[stage_name] = {
       node("canvas-$test_label-docker") {
+        def duration = TimeCategory.minus(new Date(), timeStart).toMilliseconds()
         // make sure to unstash
         unstash name: "build-dir"
         unstash name: "build-docker-compose"
+        load('build/new-jenkins/groovy/splunk.groovy').uploadEvent('jenkins.node.wait', ['duration': duration, 'node': stage_name])
         def extraTags = ["parallelStageName:${stage_name}"]
         runDatadogMetric(test_label,extraTags) {
           stage_block(index)

build/new-jenkins/groovy/splunk.groovy

@@ -16,10 +16,10 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-import groovy.json.*
-
 // https://docs.splunk.com/Documentation/Splunk/8.0.3/Data/FormateventsforHTTPEventCollector#Event_metadata
 
+import groovy.json.*
+
 def upload(events) {
   def data = events.collect { new JsonBuilder(it).toString() }.join('')
   load('build/new-jenkins/groovy/credentials.groovy').withSplunkCredentials({
@@ -35,4 +35,8 @@ def event(name, fields) {
   ]
 }
 
+def uploadEvent(name, fields) {
+  upload([event(name, fields)])
+}
+
 return this