Add a way to generate better js coverage

This should pick up coverage from both karma and jest in Canvas proper. It should also pick up coverage from anything under packages/* closes CORE-1265 Test Plan: - Run `COVERAGE=true yarn test` - Coverage reports (coverage-final.json) should generate in all packages such as (canvas-planner) as well as in coverage-karma and coverage-jest - Run `yarn run test:coverage` - coverage-js folder should be generated. Opening index.html should show the full coverage report - Remove coverage-{js, karma, jest} and packages/canvas-planner/coverage as well as .nyc_output to get back to a clean state - Run `RUN_TESTS_FIRST=true yarn run test:coverage` - All tests should run and generate the appropriate report just like the last time in coverage-js Change-Id: I50744b8977e0683e079af5bdd2865dbcd6ad9066 Reviewed-on: https://gerrit.instructure.com/146098 Tested-by: Jenkins Reviewed-by: Ryan Shaw <ryan@instructure.com> QA-Review: Jeremy Putnam <jeremyp@instructure.com> Product-Review: Clay Diffrient <cdiffrient@instructure.com>
2018-04-05 15:27:42 -06:00 · 2018-04-05 15:27:42 -06:00 · 2dc1d7fa86
parent 4e012e178e
commit 2dc1d7fa86
9 changed files with 183 additions and 20 deletions
--- a/.gitignore
+++ b/.gitignore
@ -73,3 +73,11 @@ docker-compose.local.*

 # generated scope names
 /lib/api_scope_mapper.rb
+
+# generated scopes markdown
+/doc/api/api_token_scopes.md
+
+# js coverage stuff
+.nyc_output
+coverage-jest
+coverage-karma
--- a/doc/js_code_coverage.md
+++ b/doc/js_code_coverage.md
@ -0,0 +1,94 @@
+# JavaScript Code Coverage
+
+**tl;dr** - If you want to generate a single report for all the code run `RUN_TESTS_FIRST=true yarn run test:coverage`
+
+Because we use several test frameworks, we need to approach code coverage a bit differently.
+
+If you run:
+
+```bash
+
+COVERAGE=true yarn test
+
+```
+
+then you will generate coverage reports for the entire canvas-lms JavaScript codebase including qUnit tests located in
+spec/*, jest tests colocated with the app code, and any modules within the canvas-lms/packages directory.
+**Note that this will only generate individual coverage reports, not a combined report.**
+
+## Generate Combined Coverage Report
+
+If you want to generate a combined coverage report containing all the information from all the various coverage reports,
+then you can do so by running:
+
+```bash
+
+yarn run test:coverage
+
+```
+
+This will generate an HTML report with combined data from all the tests and output it to the `coverage-js`
+directory.
+
+**Note however, that this requires you to have previously generated individual coverage reports.**
+
+If you want to ensure that all the coverage reports are generated beforehand, then you want to run:
+
+```bash
+
+RUN_TESTS_FIRST=true yarn run test:coverage
+
+```
+
+which will call `COVERAGE=true yarn test` prior to doing the report.
+
+
+## Canvas Jest Coverage
+
+If you run:
+
+```bash
+
+yarn run test:jest:coverage
+
+```
+
+then you will run the jest tests for Canvas and the coverage report will be placed in the `coverage-jest` directory.
+
+## Canvas qUnit/Karma Coverage
+
+If you run:
+
+```bash
+
+COVERAGE=true yarn run test:karma
+
+```
+
+then you will run the qUnit/Karma tests for Canvas and the coverage report will be placed in the `coverage-karma` directory.
+
+## Packages Coverage
+
+If you run:
+
+```bash
+
+COVERAGE=true yarn run test:packages
+
+```
+
+then you will generate coverage reports for each package.
+
+
+
+## Setting up a package/* for coverage
+
+We make the assumption that you have a `test:coverage` script defined in your package.json.  This script should
+generate a coverage report in the `json` format.  Check out https://istanbul.js.org/docs/advanced/alternative-reporters/#json)
+for more details on what that should look like.  This report should be output to a `coverage` directory at the root of the
+package.
+
+
+## Caveats
+
+Computers are really good at helping us out but sometimes get it wrong.  If for some reason you are seeing issues, try clearing out the coverage data and starting over.
--- a/jest.config.js
+++ b/jest.config.js
@ -18,6 +18,9 @@ module.exports = {
  testMatch: [
    '**/__tests__/**/?(*.)(spec|test).js'
  ],
+
+  coverageDirectory: '<rootDir>/coverage-jest/',
+
  transform: {
    '^i18n': '<rootDir>/jest/i18nTransformer.js',
    '^.+\\.jsx?$': 'babel-jest'
--- a/karma.conf.js
+++ b/karma.conf.js
@ -85,15 +85,15 @@ const karmaConfig = {
 if (process.env.COVERAGE) {
  karmaConfig.reporters.push('coverage-istanbul')
  karmaConfig.coverageIstanbulReporter = {
-    reports: ['html'],
-    dir: 'coverage-js/',
+    reports: ['html', 'json'],
+    dir: 'coverage-karma/',
    fixWebpackSourcePaths: true
  }
  karmaConfig.webpack.module.rules.unshift({
    test: /\.(js|coffee)$/,
    use: {
      loader: 'istanbul-instrumenter-loader',
-      options: { esModules: true }
+      options: { esModules: true, produceSourceMap: true }
    },
    enforce: 'post',
    exclude: /(node_modules|spec|public\/javascripts\/(bower|client_apps|translations|vendor|custom_moment_locales|custom_timezone_locales))/,
--- a/package.json
+++ b/package.json
@ -156,6 +156,7 @@
    "happypack": "^3.1.0",
    "imports-loader": "^0.8",
    "istanbul-instrumenter-loader": "^3",
+    "istanbul-merge": "^1.1.1",
    "jest": "^23",
    "jest-junit": "^5",
    "jest-localstorage-mock": "^2",
@ -174,6 +175,7 @@
    "merge-stream": "^1",
    "moxios": "^0.4",
    "npm-run-all": "^4.1.3",
+    "nyc": "^12",
    "prettier": "^1",
    "qunitjs": "^1.14.0",
    "raven-js": "^3.26.2",
@ -199,16 +201,18 @@
  },
  "repository": "instructure/canvas-lms",
  "scripts": {
-    "test": "concurrently --names \"packages,jest,karma\" \"yarn test:packages\" \"jest\" \"if [ \"$COVERAGE\" ]; then yarn test:karma ; else yarn test:karma:concurrently; fi\"",
+    "test": "concurrently --names \"packages,jest,karma\" \"yarn test:packages\" \"if [ \"$COVERAGE\" ]; then yarn test:jest:coverage; else yarn test:jest; fi\" \"if [ \"$COVERAGE\" ]; then yarn test:karma; else yarn test:karma:concurrently; fi\"",
+    "test:coverage": "script/generate_js_coverage",
    "test:watch": "concurrently --names \"jest,karma\" \"jest --watch .\" \"yarn test:karma:watch\"",
-    "test:jest": "echo 'this is here just so you know you can use jest, but just use the `jest` cli directly' && jest --color",
+    "test:jest": "jest --color",
+    "test:jest:coverage": "yarn run test:jest --coverage",
    "test:jest:debug": "node --inspect node_modules/.bin/jest --runInBand --watch",
    "test:karma": "yarn run test:karma:watch --single-run",
    "test:karma:concurrently": "concurrently --names \"coffee,js1,js2\" \"JSPEC_GROUP=coffee yarn test:karma:headless\" \"JSPEC_GROUP=js1 yarn test:karma:headless\" \"JSPEC_GROUP=js2 yarn test:karma:headless\"",
    "test:karma:headless": "yarn run test:karma --browsers ChromeHeadlessNoSandbox",
    "test:karma:watch": "node --max-old-space-size=4096 ./node_modules/.bin/karma start",
    "test:karma:watch:headless": "yarn run test:karma:watch --browsers ChromeHeadlessNoSandbox",
-    "test:packages": "yarn workspace-run test",
+    "test:packages": "if [ \"$COVERAGE\" ]; then yarn workspace-run test:coverage; else yarn workspace-run test; fi",
    "build": "yarn run build:css && yarn run build:js",
    "build:watch": "concurrently \"yarn build:css:watch\" \"yarn build:js:watch\"",
    "build:css": "brandable_css",
--- a/packages/canvas-planner/jest.config.js
+++ b/packages/canvas-planner/jest.config.js
@ -35,7 +35,8 @@ module.exports = {
  testRegex: "/__tests__/.*\\.(test|spec)\\.js$",
  coverageReporters: [
    'html',
-    'text'
+    'text',
+    'json'
  ],
  collectCoverageFrom: [
    'src/**/*.js'
--- a/packages/canvas-rce/package.json
+++ b/packages/canvas-rce/package.json
@ -11,7 +11,7 @@
    "lint:fix": "eslint --fix \"src/**/*.js\" \"app/**/*.js\" \"test/**/*.js\" \"shared/**/*.js\"",
    "test": "BABEL_ENV=test-node mocha 'test/**/*.test.js' --require @instructure/ui-themes/lib/canvas --compilers js:babel-core/register --timeout 5000",
    "test:watch": "BABEL_ENV=test-node mocha 'test/**/*.test.js' --require @instructure/ui-themes/lib/canvas --compilers js:babel-core/register --watch",
-    "test-cov": "cross-env BABEL_ENV=test-node nyc -r html  node_modules/.bin/mocha -- 'test/**/*.test.js'",
+    "test:coverage": "cross-env BABEL_ENV=test-node nyc -r html -r json node_modules/.bin/mocha -- 'test/**/*.test.js'",
    "debug": "BABEL_ENV=test-node inspect _mocha --no-timeouts --debug-brk 'test/**/*.test.js' --require @instructure/ui-themes/lib/canvas --compilers js:babel-core/register",
    "demo": "webpack -p",
    "dev": "webpack-dev-server -d --content-base github-pages/",
--- a/script/generate_js_coverage
+++ b/script/generate_js_coverage
@ -0,0 +1,22 @@
+#!/bin/bash
+
+###
+# This script does the work of combining various coverage JS code coverage files
+# together to produce one common html (and json) coverage report for the entire
+# repository, including the packages we've separated out in our packages/ directory.
+#
+#
+###
+
+export COVERAGE=true
+
+if [ "$RUN_TESTS_FIRST" ]; then
+    echo '========== Running tests with coverage ==========='
+    yarn test
+fi
+
+echo '========== Merging all coverage reports =========='
+./node_modules/.bin/istanbul-merge --out .nyc_output/total-coverage.json "{coverage-karma,coverage-jest,packages/*/coverage}/**/coverage*.json"
+
+echo '============= Generating HTML Report ============='
+./node_modules/.bin/nyc report --reporter=html --report-dir coverage-js
--- a/yarn.lock
+++ b/yarn.lock
@ -8524,7 +8524,7 @@ istanbul-instrumenter-loader@^3:
    loader-utils "^1.1.0"
    schema-utils "^0.3.0"

-istanbul-lib-coverage@^1.1.2, istanbul-lib-coverage@^1.2.0:
+istanbul-lib-coverage@^1.0.0, istanbul-lib-coverage@^1.2.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-1.2.0.tgz#f7d8f2e42b97e37fe796114cb0f9d68b5e3a4341"

@ -8568,16 +8568,7 @@ istanbul-lib-instrument@^2.1.0:
    istanbul-lib-coverage "^2.0.0"
    semver "^5.5.0"

-istanbul-lib-report@^1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-1.1.3.tgz#2df12188c0fa77990c0d2176d2d0ba3394188259"
-  dependencies:
-    istanbul-lib-coverage "^1.1.2"
-    mkdirp "^0.5.1"
-    path-parse "^1.0.5"
-    supports-color "^3.1.2"
-
-istanbul-lib-report@^1.1.4:
+istanbul-lib-report@^1.1.3, istanbul-lib-report@^1.1.4:
  version "1.1.4"
  resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-1.1.4.tgz#e886cdf505c4ebbd8e099e4396a90d0a28e2acb5"
  dependencies:
@ -8596,6 +8587,16 @@ istanbul-lib-source-maps@^1.2.4, istanbul-lib-source-maps@^1.2.5:
    rimraf "^2.6.1"
    source-map "^0.5.3"

+istanbul-merge@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/istanbul-merge/-/istanbul-merge-1.1.1.tgz#75be643436d4de723e03a94d1ca5f5f994b0bd72"
+  dependencies:
+    foreach "^2.0.5"
+    glob "^7.0.5"
+    istanbul-lib-coverage "^1.0.0"
+    mkdirp "^0.5.1"
+    yargs "^4.8.1"
+
 istanbul-reports@^1.3.0:
  version "1.3.0"
  resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-1.3.0.tgz#2f322e81e1d9520767597dca3c20a0cce89a3554"
@ -9740,7 +9741,7 @@ lodash._root@^3.0.0:
  version "3.0.1"
  resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692"

-lodash.assign@^4.2.0:
+lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.2.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"

@ -16006,6 +16007,10 @@ window-size@^0.1.4:
  version "0.1.4"
  resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.4.tgz#f8e1aa1ee5a53ec5bf151ffa09742a6ad7697876"

+window-size@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
+
 winston@2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/winston/-/winston-2.1.0.tgz#34688215cc8dbb784838b9aa626e73aee44fe4b6"
@ -16220,6 +16225,13 @@ yargs-parser@^10.0.0:
  dependencies:
    camelcase "^4.1.0"

+yargs-parser@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4"
+  dependencies:
+    camelcase "^3.0.0"
+    lodash.assign "^4.0.6"
+
 yargs-parser@^4.2.0:
  version "4.2.1"
  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-4.2.1.tgz#29cceac0dc4f03c6c87b4a9f217dd18c9f74871c"
@ -16335,6 +16347,25 @@ yargs@^3.5.4, yargs@^3.8.0:
    window-size "^0.1.4"
    y18n "^3.2.0"

+yargs@^4.8.1:
+  version "4.8.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-4.8.1.tgz#c0c42924ca4aaa6b0e6da1739dfb216439f9ddc0"
+  dependencies:
+    cliui "^3.2.0"
+    decamelize "^1.1.1"
+    get-caller-file "^1.0.1"
+    lodash.assign "^4.0.3"
+    os-locale "^1.4.0"
+    read-pkg-up "^1.0.1"
+    require-directory "^2.1.1"
+    require-main-filename "^1.0.1"
+    set-blocking "^2.0.0"
+    string-width "^1.0.1"
+    which-module "^1.0.0"
+    window-size "^0.2.0"
+    y18n "^3.2.1"
+    yargs-parser "^2.4.1"
+
 yargs@^7.0.0:
  version "7.1.0"
  resolved "https://registry.yarnpkg.com/yargs/-/yargs-7.1.0.tgz#6ba318eb16961727f5d284f8ea003e8d6154d0c8"