canvas_i18nliner: implement ESM extractor

refs FOO-2696
flag = none

this new extractor is tailored for ES modules away from the previous AMD
implementation. It will be put to use once we add support for the
useScope interface in @canvas/i18n and adjust webpack/source files to
use it.

This is how i18n extraction works for ESM:

(1) import the "useScope" function from @canvas/i18n

    import { useScope } from '@canvas/i18n'
    import { useScope as useI18nScope } from '@canvas/i18n' // ALSO OK

(2) use that function to define your I18n receiver and supply a scope:

    const I18n = useScope('foo')

(3) proceed to call I18n.t or I18n.translate as usual:

    I18n.t('my_key', 'Hello') // => foo.my_key

the implementation required an upstream change to i18nliner-js; see
4040b1c979

~ test plan ~
  ---- ----

- read through the extractor code and rely on the tests since it's not
  wired yet

Change-Id: I5c1ff0c23983c0e61a649c9fb3f4673724d6e468
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/286652
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Charley Kline <ckline@instructure.com>
Product-Review: Charley Kline <ckline@instructure.com>
QA-Review: Ahmad Amireh <ahmad@instructure.com>

gems/canvas_i18nliner/js/errors.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019 - present Instructure, Inc.
+ * Copyright (C) 2022 - present Instructure, Inc.
  *
  * This file is part of Canvas.
  *
@@ -16,10 +16,8 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-var HtmlReporter = require('jasmine-pretty-html-reporter').Reporter;
+const { default: Errors } = require("@instructure/i18nliner/dist/lib/errors");
-var path = require('path');
 
-// options object
+Errors.register("UnscopedTranslateCall");
-jasmine.getEnv().addReporter(new HtmlReporter({
+
-  path: path.join(__dirname,'../../../../tmp/spec_results')
+module.exports = Errors
-}));

gems/canvas_i18nliner/js/scoped_esm_extractor.js

@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2022 - present Instructure, Inc.
+ *
+ * This file is part of Canvas.
+ *
+ * Canvas is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, version 3 of the License.
+ *
+ * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const createScopedTranslateCall = require("./scoped_translate_call")
+const Errors = require("./errors");
+const { default: I18nJsExtractor } = require("@instructure/i18nliner/dist/lib/extractors/i18n_js_extractor");
+const { default: TranslateCall } = require("@instructure/i18nliner/dist/lib/extractors/translate_call");
+
+const ScopedTranslateCall = createScopedTranslateCall(TranslateCall);
+const CANVAS_I18N_PACKAGE = '@canvas/i18n'
+const CANVAS_I18N_USE_SCOPE_SPECIFIER = 'useScope'
+const CANVAS_I18N_RECEIVER = 'I18n'
+
+// This extractor implementation is suitable for ES modules where a module
+// imports the "useScope" function from the @canvas/i18n package and assigns the
+// output of a call to that function to a receiver named exactly "I18n". Calls
+// to the "t" or "translate" methods on that receiver will use the scope
+// supplied to that "useScope" call.
+//
+//     import { useScope } from '@canvas/i18n'
+//
+//     const I18n = useScope('foo')
+//
+//     I18n.t('my_key', 'Hello world!')
+//     // => { "foo": { "my_key": "Hello World" } }
+//
+// The extractor looks for the I18n receiver defined in the current lexical
+// scope of the call to I18n.t():
+//
+//     function a() {
+//       const I18n = useScope('foo')
+//       I18n.t('my_key', 'Key in foo') // => foo.my_key
+//     }
+//
+//     function b() {
+//       const I18n = useScope('bar')
+//       I18n.t('my_key', 'Key in bar') // => bar.my_key
+//     }
+//
+// Note that the receiver MUST be identified as "I18n". The (base) extractor
+// will fail to recognize any translate calls if the output of useScope is
+// assigned to a receiver with a different identifier. With that said, the
+// identifier for useScope can be renamed at will:
+//
+//     // this is OK:
+//     import { useScope as useI18nScope } from '@canvas/i18n'
+//     const I18n = useI18nScope('foo')
+//
+//     // this is NOT ok:
+//     import { useScope } from '@canvas/i18n'
+//     const smth = useScope('foo')
+//
+class ScopedESMExtractor extends I18nJsExtractor {
+  constructor() {
+    super(...arguments)
+
+    // the identifier for the "useScope" specifier imported from @canvas/i18n,
+    // which may be renamed
+    this.useScopeIdentifier = null
+
+    // mapping of "I18n" receivers to the (i18n) scopes they were assigned in
+    // the call to useScope
+    this.receiverScopeMapping = new WeakMap()
+  };
+
+  enter(path) {
+    // import { useScope } from '@canvas/i18n'
+    //          ^^^^^^^^
+    // import { useScope as blah } from '@canvas/i18n'
+    //                      ^^^^
+    if (!this.useScopeIdentifier && path.type === 'ImportDeclaration') {
+      trackUseScopeIdentifier.call(this, path);
+    }
+    // let I18n
+    //     ^^^^
+    // I18n = useScope('foo')
+    //                  ^^^
+    // (this happens in CoffeeScript when compiled to JS)
+    else if (this.useScopeIdentifier && path.type === 'AssignmentExpression') {
+      indexScopeFromAssignment.call(this, path)
+    }
+    // const I18n = useScope('foo')
+    //       ^^^^             ^^^
+    else if (this.useScopeIdentifier && path.type === 'VariableDeclarator') {
+      indexScopeFromDeclaration.call(this, path)
+    }
+
+    return super.enter(...arguments)
+  };
+
+  buildTranslateCall(line, method, args, path) {
+    const binding = path.scope.getBinding(CANVAS_I18N_RECEIVER)
+    const scope = this.receiverScopeMapping.get(binding)
+
+    if (scope) {
+      return new ScopedTranslateCall(line, method, args, scope);
+    }
+    else {
+      throw new Errors.UnscopedTranslateCall(line)
+    }
+  };
+};
+
+function trackUseScopeIdentifier({ node }) {
+  if (
+    node.source &&
+    node.source.type === 'StringLiteral' &&
+    node.source.value === CANVAS_I18N_PACKAGE
+  ) {
+    const specifier = node.specifiers.find(x =>
+      x.type === 'ImportSpecifier'&&
+      x.imported &&
+      x.imported.type === 'Identifier' &&
+      x.imported.name === CANVAS_I18N_USE_SCOPE_SPECIFIER
+    )
+
+    if (
+      specifier &&
+      specifier.type === 'ImportSpecifier' &&
+      specifier.local &&
+      specifier.local.type === 'Identifier' &&
+      specifier.local.name
+    ) {
+      this.useScopeIdentifier = specifier.local.name
+    }
+  }
+};
+
+function indexScopeFromAssignment(path) {
+  return indexScope.call(this, path, path.node.left, path.node.right)
+};
+
+function indexScopeFromDeclaration(path) {
+  return indexScope.call(this, path, path.node.id, path.node.init)
+};
+
+// left: Identifier
+// right: CallExpression
+function indexScope(path, left, right) {
+  if (
+    left &&
+    left.type === 'Identifier' &&
+    left.name === CANVAS_I18N_RECEIVER &&
+    right &&
+    right.type === 'CallExpression' &&
+    right.callee &&
+    right.callee.type === 'Identifier' &&
+    right.callee.name === this.useScopeIdentifier &&
+    right.arguments &&
+    right.arguments.length === 1 &&
+    right.arguments[0].type === 'StringLiteral' &&
+    right.arguments[0].value
+  ) {
+    this.receiverScopeMapping.set(
+      path.scope.getBinding(CANVAS_I18N_RECEIVER),
+      right.arguments[0].value
+    )
+  }
+};
+
+module.exports = ScopedESMExtractor;

gems/canvas_i18nliner/js/scoped_i18n_js_extractor.js

@@ -16,9 +16,7 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-var Errors = require("@instructure/i18nliner/dist/lib/errors").default;
+var Errors = require("./errors");
-Errors.register("UnscopedTranslateCall");
-
 var TranslateCall = require("@instructure/i18nliner/dist/lib/extractors/translate_call").default;
 var ScopedTranslateCall = require("./scoped_translate_call")(TranslateCall);
 

gems/canvas_i18nliner/package.json

@@ -15,11 +15,10 @@
   },
   "devDependencies": {
     "coffee-script": "^1.12.4",
-    "jasmine": "^2.5.3",
+    "jest": "^26"
-    "jasmine-pretty-html-reporter": "^0.2.5"
   },
   "peerDependencies": {},
   "scripts": {
-    "test": "./node_modules/.bin/jasmine"
+    "test": "jest"
   }
 }

gems/canvas_i18nliner/spec/fixtures/js/app/esm/esm.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2022 - present Instructure, Inc.
+ *
+ * This file is part of Canvas.
+ *
+ * Canvas is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, version 3 of the License.
+ *
+ * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import I18n from 'i18n!esm'
+
+I18n.t('my_key', 'Hello world')

gems/canvas_i18nliner/spec/i18nliner.test.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015 - present Instructure, Inc.
+ * Copyright (C) 2022 - present Instructure, Inc.
  *
  * This file is part of Canvas.
  *
@@ -20,8 +20,23 @@ const mkdirp = require("mkdirp");
 const { I18nliner } = require("../js/main");
 const scanner = require("../js/scanner");
 
+class PanickyCheck extends I18nliner.Commands.Check {
+  // don't print to TTY
+  print() {};
+
+  // and do throw errors
+  checkWrapper(file, checker) {
+    try {
+      checker(file)
+    }
+    catch (e) {
+      throw new Error(e.message)
+    }
+  };
+};
+
 var subject = function(dir) {
-  var command = new I18nliner.Commands.Check({});
+  var command = new PanickyCheck({});
   scanner.scanFilesFromI18nrc(scanner.loadConfigFromDirectory(dir))
   command.run();
   return command.translations.masterHash.translations;
@@ -52,15 +67,15 @@ describe("I18nliner", function() {
     });
 
     it('throws if no scope was specified', () => {
-      const command = new I18nliner.Commands.Check({});
+      const command = new PanickyCheck({});
-      const origDir = process.cwd();
 
-      scanner.scanFilesFromI18nrc(scanner.loadConfigFromDirectory('spec/fixtures/hbs-missing-i18n-scope'))
+      scanner.scanFilesFromI18nrc(
-      command.checkFiles();
+        scanner.loadConfigFromDirectory('spec/fixtures/hbs-missing-i18n-scope')
+      )
 
-      expect(command.isSuccess()).toBeFalsy()
+      expect(() => {
-      expect(command.errors.length).toEqual(1)
+        command.checkFiles()
-      expect(command.errors[0]).toMatch(/expected i18nScope for Handlebars template to be specified/)
+      }).toThrowError(/expected i18nScope for Handlebars template to be specified/)
     })
   });
 
@@ -69,6 +84,9 @@ describe("I18nliner", function() {
       expect(subject("spec/fixtures/js")).toEqual({
         absolute_key: "Absolute key",
         inferred_key_c49e3743: "Inferred key",
+        esm: {
+          my_key: 'Hello world'
+        },
         foo: {
           relative_key: "Relative key"
         },

gems/canvas_i18nliner/spec/scoped_esm_extractor.test.js

@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) 2022 - present Instructure, Inc.
+ *
+ * This file is part of Canvas.
+ *
+ * Canvas is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, version 3 of the License.
+ *
+ * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const JsProcessor = require('i18nliner/dist/lib/processors/js_processor')['default'];
+const ScopedESMExtractor = require('../js/scoped_esm_extractor');
+const dedent = require('dedent')
+
+describe('ScopedESMExtractor', () => {
+  it('tracks scope through the call to @canvas/i18n#useScope', () => {
+    expect(extract(dedent`
+      import { useScope } from '@canvas/i18n'
+      const I18n = useScope('foo')
+      I18n.t('keyed', 'something')
+    `).translations.translations).toEqual({
+      foo: {
+        keyed: 'something'
+      }
+    });
+  })
+
+  it('tracks scope through the call to a renamed @canvas/i18n#useScope specifier', () => {
+    expect(extract(dedent`
+      import { useScope as useI18nScope } from '@canvas/i18n'
+      const I18n = useI18nScope('foo')
+      I18n.t('keyed', 'something')
+    `).translations.translations).toEqual({
+      foo: {
+        keyed: 'something'
+      }
+    });
+  })
+
+  it('tracks scope through assignment', () => {
+    expect(extract(dedent`
+      import { useScope } from '@canvas/i18n'
+
+      let I18n
+
+      I18n = useScope('foo')
+      I18n.t('keyed', 'something')
+    `).translations.translations).toEqual({
+      foo: {
+        keyed: 'something'
+      }
+    });
+  })
+
+  it('resolves (i18n) scopes across different lexical scopes', () => {
+    const { translations: translationHash } = extract(dedent`
+      import { useScope } from '@canvas/i18n'
+
+      function a() {
+        const I18n = useScope('foo')
+
+        I18n.t('key', 'hello')
+        I18n.t('inferred')
+      }
+
+      function b() {
+        let I18n
+
+        I18n = useScope('bar')
+        I18n.t('key', 'world')
+        I18n.t('inferred')
+      }
+    `)
+
+    expect(translationHash.translations).toEqual({
+      foo: { key: 'hello' },
+      bar: { key: 'world' },
+      inferred_7cf5962e: 'inferred'
+    })
+  })
+
+  it('extracts translations', () => {
+    expect(extract(dedent`
+      import { useScope } from '@canvas/i18n'
+
+      const I18n = useScope('foo')
+
+      I18n.t('#absolute', 'Unscoped')
+      I18n.t('inferred')
+      I18n.t('keyed', 'Keyed')
+      I18n.t('nested.keyed', 'Nested')
+    `).translations.translations).toEqual({
+      absolute: 'Unscoped',
+      foo: {
+        keyed: 'Keyed',
+        nested: {
+          keyed: 'Nested'
+        }
+      },
+      inferred_7cf5962e: 'inferred'
+    });
+  })
+
+  it('throws if no scope was defined by the time t() was called', () => {
+    expect(() => {
+      extract(dedent`
+        import I18n from '@canvas/i18n'
+        I18n.t('hello')
+      `)
+    }).toThrow(/unscoped translate call/);
+  })
+
+  it('throws if no scope was defined using the "useScope" specifier', () => {
+    expect(() => {
+      extract(dedent`
+        import { useScope } from '@canvas/i18n'
+        const I18n = somethingElse('foo')
+        I18n.t('hello')
+      `)
+    }).toThrow(/unscoped translate call/);
+  })
+
+  it('throws if no scope was defined using the "useScope" interface from @canvas/i18n', () => {
+    expect(() => {
+      extract(dedent`
+        function useScope() {}
+        const I18n = useScope('foo')
+        I18n.t('hello')
+      `)
+    }).toThrow(/unscoped translate call/);
+  })
+});
+
+function extract(source) {
+  const extractor = new ScopedESMExtractor({
+    ast: JsProcessor.prototype.parse(source)
+  })
+
+  extractor.run()
+
+  return extractor;
+}

gems/canvas_i18nliner/spec/support/jasmine.json

@@ -1,11 +0,0 @@
-{
-  "spec_dir": "spec",
-  "spec_files": [
-    "**/*[sS]pec.js"
-  ],
-  "helpers": [
-    "helpers/**/*.js"
-  ],
-  "stopSpecOnExpectationFailure": false,
-  "random": false
-}

yarn.lock

@@ -13069,7 +13069,7 @@ glob@^5.0.15:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.0.6, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
+glob@^7, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
   version "7.1.7"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90"
   integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==
@@ -15393,25 +15393,6 @@ iterate-value@^1.0.2:
     es-get-iterator "^1.0.2"
     iterate-iterator "^1.0.1"
 
-jasmine-core@~2.99.0:
-  version "2.99.1"
-  resolved "https://registry.yarnpkg.com/jasmine-core/-/jasmine-core-2.99.1.tgz#e6400df1e6b56e130b61c4bcd093daa7f6e8ca15"
-  integrity sha1-5kAN8ea1bhMLYcS80JPap/boyhU=
-
-jasmine-pretty-html-reporter@^0.2.5:
-  version "0.2.5"
-  resolved "https://registry.yarnpkg.com/jasmine-pretty-html-reporter/-/jasmine-pretty-html-reporter-0.2.5.tgz#c61b7528bf06df386d5ef4360d7d029149692bc1"
-  integrity sha1-xht1KL8G3zhtXvQ2DX0CkUlpK8E=
-
-jasmine@^2.5.3:
-  version "2.99.0"
-  resolved "https://registry.yarnpkg.com/jasmine/-/jasmine-2.99.0.tgz#8ca72d102e639b867c6489856e0e18a9c7aa42b7"
-  integrity sha1-jKctEC5jm4Z8ZImFbg4YqceqQrc=
-  dependencies:
-    exit "^0.1.2"
-    glob "^7.0.6"
-    jasmine-core "~2.99.0"
-
 jest-canvas-mock@^2:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/jest-canvas-mock/-/jest-canvas-mock-2.3.1.tgz#9535d14bc18ccf1493be36ac37dd349928387826"