ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add rspec coverage to main-for-coverage build 

fixes: CCI-170
flag = none

Test-Plan:
manually main-for-coverage with this specref and ensure
coverage is uploaded

Change-Id: Ifc67a65157e8ad5853b01664fe81e177f8bba0d6
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/226086
QA-Review: Rex Fleischer <rfleischer@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Jacob Powell <spowell@instructure.com>
Reviewed-by: James Butters <jbutters@instructure.com>
Product-Review: Rex Fleischer <rfleischer@instructure.com>
This commit is contained in:
Rex Fleischer 2020-02-10 10:12:41 -08:00
parent b479c42e60
commit 527ec9c6ed
7 changed files with 140 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Jenkinsfile.main-for-coverage
View File

@ -61,16 +61,14 @@ pipeline {
// }
// }
// stage('Rspec') {
// steps {
// skipIfPreviouslySuccessful("rspec") {
// build(
// job: 'test-suites/rspec',
// parameters: build_parameters
// )
// }
// }
// }
stage('Rspec') {
steps {
build(
job: 'test-suites/rspec',
parameters: build_parameters
)
}
}
}
}
}

38
Jenkinsfile.rspec
View File

@ -18,6 +18,16 @@
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
def runCoverage() {
def flags = load 'build/new-jenkins/groovy/commit-flags.groovy'
return env.RUN_COVERAGE == '1' || flags.forceRunCoverage() ? '1' : ''
}
def getImageTagVersion() {
def flags = load 'build/new-jenkins/groovy/commit-flags.groovy'
return env.RUN_COVERAGE == '1' ? 'master' : flags.getImageTagVersion()
}
def ci_node_total = 15; // how many nodes to run on
pipeline {
agent { label 'canvas-docker' }
@ -27,8 +37,7 @@ pipeline {
environment {
COMPOSE_FILE = 'docker-compose.new-jenkins.yml'
// 'refs/changes/63/181863/8' -> '63.181863.8'
NAME = "${env.GERRIT_REFSPEC}".minus('refs/changes/').replaceAll('/','.')
NAME = getImageTagVersion()
PATCHSET_TAG = "$DOCKER_REGISTRY_FQDN/jenkins/canvas-lms:$NAME"
KNAPSACK_ENABLED = 1
KNAPSACK_GENERATE_REPORT = 'false'
@ -37,6 +46,7 @@ pipeline {
KNAPSACK_TEST_DIR = 'spec'
RERUNS_RETRY = 1
MAX_FAIL = 50
COVERAGE = runCoverage()
}
stages {
stage ('Distribute Rspec Tests') {
@ -46,13 +56,14 @@ pipeline {
for(int i = 0; i < ci_node_total; i++) {
def index = i;
nodes["rspec set ${(i).toString().padLeft(2, '0')}"] = {
withEnv(["CI_NODE_INDEX=$index", "CI_NODE_TOTAL=$ci_node_total"]) {
withEnv(["TEST_ENV_NUMBER=$index", "CI_NODE_INDEX=$index", "CI_NODE_TOTAL=$ci_node_total"]) {
node('canvas-docker') {
stage("Running RSpec Set ${index}") {
try {
sh 'rm -rf ./tmp/spec_failures'
sh 'rm -rf ./tmp'
checkout scm
sh 'build/new-jenkins/docker-cleanup.sh'
sh 'mkdir -p tmp'
timeout(time: 60) {
sh 'build/new-jenkins/print-env-excluding-secrets.sh'
sh 'build/new-jenkins/docker-compose-pull.sh'
@ -63,16 +74,17 @@ pipeline {
}
catch (ex) {
// copy spec failures to local
sh 'mkdir -p tmp'
sh 'docker cp $(docker-compose ps -q web):/usr/src/app/log/spec_failures/ ./tmp/spec_failures/'
throw ex
}
finally {
def reports = load 'build/new-jenkins/groovy/reports.groovy'
dir ('tmp') {
reports.stashSpecFailures(index)
reports.stashSpecFailures(index)
if (env.COVERAGE == '1') {
sh 'docker cp $(docker-compose ps -q web):/usr/src/app/coverage/ ./tmp/spec_coverage/'
reports.stashSpecCoverage(index)
}
sh 'rm -rf ./tmp/spec_failures'
sh 'rm -rf ./tmp'
sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
}
}
@ -84,6 +96,16 @@ pipeline {
}
}
}
stage('Upload Coverage') {
when { expression { env.COVERAGE == '1' } }
steps {
script {
def reports = load 'build/new-jenkins/groovy/reports.groovy'
reports.publishSpecCoverageToS3(ci_node_total)
}
}
}
}
post {
failure {

4
Jenkinsfile.selenium.chrome
View File

@ -72,9 +72,7 @@ pipeline {
}
finally {
def reports = load 'build/new-jenkins/groovy/reports.groovy'
dir ('tmp') {
reports.stashSpecFailures(index)
}
reports.stashSpecFailures(index)
sh 'rm -rf ./tmp/spec_failures'
sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
}

35
build/new-jenkins/groovy/reports.groovy
View File

@ -16,18 +16,47 @@
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
def stashSpecCoverage(index) {
dir("tmp") {
stash name: "spec_coverage_${index}", includes: 'spec_coverage/**/*'
}
}
def publishSpecCoverageToS3(ci_node_total) {
sh 'rm -rf ./coverage_nodes'
dir('coverage_nodes') {
for(int index = 0; index < ci_node_total; index++) {
dir("node_${index}") {
unstash "spec_coverage_${index}"
}
}
}
sh './build/new-jenkins/rspec-coverage-report.sh'
archiveArtifacts(artifacts: 'coverage_nodes/**')
archiveArtifacts(artifacts: 'coverage/**')
uploadCoverage([
uploadSource: "/coverage",
uploadDest: "canvas-lms-rspec/coverage"
])
sh 'rm -rf ./coverage_nodes'
sh 'rm -rf ./coverage'
}
// this method is to ensure that the stashing is done in a way that
// is expected in publishSpecFailuresAsHTML
def stashSpecFailures(index) {
stash name: "spec_failures_${index}", includes: 'spec_failures/**/*', allowEmpty: true
dir("tmp") {
stash name: "spec_failures_${index}", includes: 'spec_failures/**/*', allowEmpty: true
}
}
def publishSpecFailuresAsHTML(ci_node_total) {
sh 'rm -rf ./compiled_failures'
def htmlFiles;
dir('compiled_failures') {
for(int i = 0; i < ci_node_total; i++) {
def index = i;
for(int index = 0; index < ci_node_total; index++) {
dir ("node_${index}") {
unstash "spec_failures_${index}"
}

31
build/new-jenkins/rspec-combine-coverage-results.py Normal file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env python3
import subprocess
import json
import os
import shutil
def fp(message):
print(message, flush=True)
def main():
results = {}
for node in os.listdir("./coverage_nodes"):
if not os.path.isdir('./coverage_nodes/{}'.format(node)):
continue
with open('./coverage_nodes/{}/spec_coverage/.resultset.json'.format(node)) as json_file:
resultset = json.load(json_file)
fp(node)
for process in resultset:
key = "{}:{}".format(node, process)
results[key] = resultset[process]
for process in results:
fp(process)
with open('./coverage_nodes/.resultset.json', 'w') as results_file:
json.dump(results, results_file)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
fp("")

37
build/new-jenkins/rspec-coverage-report.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
rm -rf coverage_reports
mkdir coverage_reports
chmod 777 coverage_reports
python3 ./build/new-jenkins/rspec-combine-coverage-results.py
# build the reports inside the canvas-lms image because it has the required executables
inputs=()
inputs+=("--volume $WORKSPACE/coverage_nodes:/usr/src/app/coverage_nodes")
inputs+=("--volume $WORKSPACE/coverage_reports:/usr/src/app/coverage_reports")
cat <<EOF | docker run --interactive ${inputs[@]} $PATCHSET_TAG /bin/bash -
set -ex
mkdir coverage
# copy the file to where the 'merger' expects it
cp coverage_nodes/.resultset.json coverage/.resultset.json
# this doesnt actually merge things, it just makes the report
bundle exec ruby spec/simple_cov_result_merger.rb
# tar this up in the mounted volume so we can get it later
tar cf coverage_reports/coverage.tar ./coverage
EOF
# extract the reports
rm -rf coverage
tar -xf coverage_reports/coverage.tar
# lets see the result after
find ./coverage_nodes
find ./coverage
rm -rf coverage_reports

1
docker-compose.new-jenkins.yml
View File

@ -18,6 +18,7 @@ services:
RANDOMIZE_SEQUENCES: 1
RERUNS_RETRY:
MAX_FAIL:
COVERAGE:
# knapsack
CI_NODE_TOTAL: