i18nliner-handlebars (part II)

extraction and runtime for ember handlebars

test plan:
1. verify string extraction:
   1. `rake js:generate i18n:generate` before and after this commit
   2. confirm `config/locales/generated/en.yml` is identical, except the
      now you get a bunch of new ember strings
2. verify js translation file generation:
   1. `rake i18n:generate_js` before and after this commit
   2. confirm the files in public/javascripts/translations are identical,
      plus a whole bunch of new ones for ember
3. confirm you can now use i18nliner-y features:
   1. block helper with no key `{{#t}}hello world{{/t}}`
   2. inline helper with no key `{{t "hello world"}}`

Change-Id: I7bca8aeba462e68f27973fa801e7f7dbc7b3c9ef
Reviewed-on: https://gerrit.instructure.com/43158
Reviewed-by: Jennifer Stern <jstern@instructure.com>
Product-Review: Jennifer Stern <jstern@instructure.com>
QA-Review: Matt Fairbourn <mfairbourn@instructure.com>
Tested-by: Jenkins <jenkins@instructure.com>
This commit is contained in:
Jon Jensen 2014-10-22 12:37:08 -06:00
parent 6d8bac05b4
commit 1300bf0352
13 changed files with 78 additions and 358 deletions

View File

@ -1,13 +1,25 @@
define ['ember', 'i18nObj'], (Ember, I18n) ->
Ember.Handlebars.registerHelper 't', (translationKey, defaultValue, options) ->
wrappers = {}
options = options?.hash ? {}
scope = options.scope
delete options.scope
for key, value of options when key.match(/^w\d+$/)
wrappers[new Array(parseInt(key.replace('w', '')) + 2).join('*')] = value
define ['ember', 'i18nObj', 'str/htmlEscape'], (Ember, I18n, htmlEscape) ->
Ember.Handlebars.registerHelper 't', (args..., hbsOptions) ->
{hash, hashTypes, hashContexts} = hbsOptions
options = {}
for own key, value of hash
type = hashTypes[key]
if type is 'ID'
options[key] = Ember.get(hashContexts[key], value)
else
options[key] = value
wrappers = []
while (key = "w#{wrappers.length}") and options[key]
wrappers.push(options[key])
delete options[key]
options.wrapper = wrappers if wrappers['*']
options.needsEscaping = true
options = Ember.$.extend(options, this) unless this instanceof String or typeof this is 'string'
I18n.scoped(scope).t(translationKey, defaultValue, options)
options.wrapper = wrappers if wrappers.length
new Ember.Handlebars.SafeString htmlEscape I18n.t(args..., options)
Ember.Handlebars.registerHelper '__i18nliner_escape', htmlEscape
Ember.Handlebars.registerHelper '__i18nliner_safe', (val) ->
new htmlEscape.SafeString(val)
Ember.Handlebars.registerHelper '__i18nliner_concat', (args..., options) ->
args.join("")

View File

@ -2,6 +2,7 @@
var readline = require('readline');
var Handlebars = require('handlebars');
var EmberHandlebars = require('ember-template-compiler').EmberHandlebars;
var ScopedHbsExtractor = require('../js/scoped_hbs_extractor');
var PreProcessor = require('i18nliner-handlebars/dist/lib/pre_processor')['default'];
@ -28,7 +29,8 @@ rl.on('line', function(line) {
PreProcessor.process(ast);
extractor.forEach(function() { translationCount++; });
var result = Handlebars.precompile(ast);
var precompiler = data.ember ? EmberHandlebars : Handlebars;
var result = precompiler.precompile(ast).toString();
var payload = {template: result, scope: scope, translationCount: translationCount};
process.stdout.write(JSON.stringify(payload) + "\n");
}

View File

@ -14,8 +14,8 @@ var glob = require("glob");
// explict subdirs, to work around perf issues
// https://github.com/jenseng/i18nliner-js/issues/7
JsProcessor.prototype.directories = ["public/javascripts"];
HbsProcessor.prototype.directories = ["app/views/jst"];
HbsProcessor.prototype.defaultPattern = "**/*.handlebars";
HbsProcessor.prototype.directories = ["app/views/jst", "app/coffeescripts/ember"];
HbsProcessor.prototype.defaultPattern = ["*.hbs", "*.handlebars"];
require("./scoped_hbs_pre_processor");
var ScopedI18nJsExtractor = require("./scoped_i18n_js_extractor");
@ -24,10 +24,10 @@ var ScopedTranslationHash = require("./scoped_translation_hash");
// remove path stuff we don't want in the scope
var pathRegex = new RegExp(
"^" + HbsProcessor.prototype.directories[0] + "(/plugins/[^/]+)?/"
"^(" + HbsProcessor.prototype.directories.join("|") + ")(/plugins/[^/]+)?/"
);
ScopedHbsExtractor.prototype.normalizePath = function(path) {
return path.replace(pathRegex, "");
return path.replace(pathRegex, "").replace(/^([^\/]+\/)templates\//, '$1');
};
var GenerateJs = require("./generate_js");

View File

@ -5,6 +5,7 @@
"main": "./js/main",
"version": "0.0.1",
"dependencies": {
"ember-template-compiler": "1.4.0",
"handlebars": "1.3.0",
"i18nliner": "0.0.15",
"i18nliner-handlebars": "0.0.11",

View File

@ -1,4 +1,5 @@
require "i18n_extraction"
require "multi_json"
require "handlebars_tasks/handlebars"
require "handlebars_tasks/ember_hbs"

View File

@ -1,23 +1,20 @@
require 'fileutils'
require 'handlebars_tasks/template_precompiler'
# Precompiles handlebars templates into JavaScript function strings
module HandlebarsTasks
class EmberHbs
class << self
include HandlebarsTasks::TemplatePrecompiler
def compile_file(path)
name = parse_name(path)
dest = parse_dest(path)
template_string = prepare_with_i18n(File.read(path), scopify(path))
precompiled = compile_template(name, template_string)
precompiled = compile_template(path)
dir = File.dirname(dest)
FileUtils.mkdir_p(dir) unless File.exists?(dir)
File.open(dest, 'w') { |f| f.write precompiled }
end
def scopify(path)
path.gsub(/^app\/coffeescripts\/ember\//, '').sub(/^_/, '').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').tr("-", "_").downcase.gsub(/\/_?/, '.')
end
def parse_dest(path)
path.gsub(/^app\/coffeescripts\/ember/, 'public/javascripts/compiled/ember').gsub(/hbs$/, 'js')
end
@ -26,33 +23,20 @@ module HandlebarsTasks
path.gsub(/^.+?\/templates\//, '').gsub(/\.hbs$/, '')
end
def prepare_with_i18n(source, scope)
@extractor = I18nExtraction::HandlebarsExtractor.new
@extractor.scan(source, :method => :gsub) do |data|
wrappers = data[:wrappers].map { |value, delimiter| " w#{delimiter.size-1}=#{value.inspect}" }.join
"{{{t #{data[:key].inspect} #{data[:value].inspect} scope=#{scope.inspect}#{wrappers}#{data[:options]}}}}"
end
end
def compile_template(path)
source = File.read(path)
name = parse_name(path)
dependencies = ['ember', 'compiled/ember/shared/helpers/common']
data = precompile_template(path, source, ember: true)
dependencies << "i18n!#{data["scope"]}" if data["translationCount"] > 0
def compile_template(name, template_string)
require "execjs"
handlebars_source = File.read(File.expand_path(File.join(__FILE__, '../../../../../', 'public/javascripts/bower/handlebars/handlebars.js')))
# execjs has no "exports" and global "var foo" does not land on "this.foo"
shims = "; this.Handlebars = Handlebars; exports = {};"
precompiler_source = File.read(File.expand_path(File.join(__FILE__, '../../../../../', 'public/javascripts/bower/ember/ember-template-compiler.js')))
context = ExecJS.compile(handlebars_source + shims + precompiler_source)
precompiled = context.eval "exports.precompile(#{template_string.inspect}).toString()", template_string
template_module = <<-END
define(['ember', 'compiled/ember/shared/helpers/common'], function(Ember) {
Ember.TEMPLATES['#{name}'] = Ember.Handlebars.template(#{precompiled});
define(#{MultiJson.dump dependencies}, function(Ember) {
Ember.TEMPLATES['#{name}'] = Ember.Handlebars.template(#{data["template"]});
});
END
template_module
end
def extract_i18n
end
end
end
end
end

View File

@ -1,10 +1,12 @@
require 'fileutils'
require 'handlebars_tasks/template_precompiler'
# Precompiles handlebars templates into JavaScript function strings
module HandlebarsTasks
class Handlebars
class << self
include HandlebarsTasks::TemplatePrecompiler
# Recursively compiles a source directory of .handlebars templates into a
# destination directory. Immitates the node.js bin script at
@ -78,7 +80,7 @@ module HandlebarsTasks
dependencies << "jst/#{require_path}"
end
data = prepare_template(id, source)
data = precompile_template(id, source)
dependencies << "i18n!#{data["scope"]}" if data["translationCount"] > 0
<<-JS
@ -92,15 +94,6 @@ define('#{plugin ? plugin + "/" : ""}jst/#{id}', #{MultiJson.dump dependencies},
JS
end
def prepare_template(path, source)
require 'json'
payload = {path: path, source: source}.to_json
compiler.puts payload
result = JSON.parse(compiler.readline)
raise result["error"] if result["error"]
result
end
def get_css(file_path)
if sass_file = Dir.glob("app/stylesheets/jst/#{file_path}.s[ac]ss").first
# renders the sass file to disk, then returns the css it wrote
@ -114,11 +107,6 @@ define('#{plugin ? plugin + "/" : ""}jst/#{id}', #{MultiJson.dump dependencies},
protected
# Returns the HBS preprocessor/compiler
def compiler
Thread.current[:hbs_compiler] ||= IO.popen("./gems/canvas_i18nliner/bin/prepare_hbs", "r+")
end
def find_partial_deps(template)
# finds partials like: {{>foo bar}} and {{>[foo/bar] baz}}
template.scan(/\{\{>\s?\[?(.+?)\]?( .*?)?}}/).map {|m| m[0].strip }.uniq

View File

@ -0,0 +1,20 @@
module HandlebarsTasks
module TemplatePrecompiler
def precompile_template(path, source, options = {})
require 'json'
payload = {path: path, source: source, ember: options[:ember]}.to_json
compiler.puts payload
result = JSON.parse(compiler.readline)
raise result["error"] if result["error"]
result
end
# Returns the HBS preprocessor/compiler
def compiler
Thread.current[:hbs_compiler] ||= begin
gempath = File.dirname(__FILE__) + "/../../.."
IO.popen("#{gempath}/canvas_i18nliner/bin/prepare_hbs", "r+")
end
end
end
end

View File

@ -3,8 +3,8 @@ require 'spec_helper'
module HandlebarsTasks
expected_precompiled_template = <<-END
define(['ember', 'compiled/ember/shared/helpers/common'], function(Ember) {
Ember.TEMPLATES['application'] = Ember.Handlebars.template(function anonymous(Handlebars,depth0,helpers,partials,data) {
define(["ember","compiled/ember/shared/helpers/common"], function(Ember) {
Ember.TEMPLATES['%s'] = Ember.Handlebars.template(function anonymous(Handlebars,depth0,helpers,partials,data) {
this.compilerInfo = [4,'>= 1.0.0'];
helpers = this.merge(helpers, Ember.Handlebars.helpers); data = data || {};
@ -19,7 +19,12 @@ helpers = this.merge(helpers, Ember.Handlebars.helpers); data = data || {};
describe EmberHbs do
describe "#compile_template" do
it "outputs a precompiled template wrapped in AMD and registers with Ember.TEMPLATES" do
EmberHbs::compile_template("application", "foo").should == expected_precompiled_template
require 'tempfile'
file = Tempfile.new("foo")
file.write "foo"
file.close
EmberHbs::compile_template(file.path).should == expected_precompiled_template % EmberHbs::parse_name(file.path)
file.unlink
end
end
@ -40,4 +45,4 @@ helpers = this.merge(helpers, Ember.Handlebars.helpers); data = data || {};
end
end
end
end
end

View File

@ -2,8 +2,3 @@ require "json"
require "active_support/all"
require "i18n_extraction/i18nliner_extensions"
module I18nExtraction
require "i18n_extraction/abstract_extractor"
require "i18n_extraction/handlebars_extractor"
end

View File

@ -1,60 +0,0 @@
module I18nExtraction
module AbstractExtractor
def initialize(options = {})
@scope = options[:scope] || ''
@translations = options[:translations] || {}
@total = 0
@total_unique = 0
super()
end
def add_translation(full_key, default, line, remove_whitespace = false)
raise "html tags on line #{line} (hint: use a wrapper or markdown)" if default =~ /<[a-z][a-z0-9]*[> \/]/i
default = default.gsub(/\s+/, ' ') if remove_whitespace
default = default.strip unless full_key =~ /separator/
@total += 1
scope = full_key.split('.')
key = scope.pop
hash = @translations
while s = scope.shift
if hash[s]
raise "#{full_key.sub((scope.empty? ? '' : '.' + scope.join('.')) + '.' + key, '').inspect} used as both a scope and a key" unless hash[s].is_a?(Hash)
else
hash[s] = {}
end
hash = hash[s]
end
if hash[key]
if hash[key] != default
if hash[key].is_a?(Hash)
raise "#{full_key.inspect} used as both a scope and a key"
else
raise "cannot reuse key #{full_key.inspect}"
end
end
else
@total_unique += 1
hash[key] = default
end
end
def infer_pluralization_hash(default)
{:one => "1 #{default}", :other => "%{count} #{default.pluralize}"}
end
def allowed_pluralization_keys
[:zero, :one, :few, :many, :other]
end
def required_pluralization_keys
[:one, :other]
end
def self.included(base)
base.instance_eval do
attr_reader :total, :total_unique
attr_accessor :translations, :scope
end
end
end
end

View File

@ -1,119 +0,0 @@
module I18nExtraction
class HandlebarsExtractor
include AbstractExtractor
I18N_CALL_START = /
\{\{
\#t \s+
(?<quote> ["'])
(?<key> .*?)
\g<quote>
(?<opts> [^\}]*)
\}\}
/x
I18N_CALL = /
#{I18N_CALL_START}
(?<content> .*?)
\{\{\/t\}\}
/mx
TAG_NAME = /[a-z][a-z0-9]*/i
TAG_START = %r{<#{TAG_NAME}[^>]*(?<!/)>}
TAG_END = %r{</#{TAG_NAME}>}
TAG_EMPTY = %r{<#{TAG_NAME}[^>]*/>}
I18N_WRAPPER = /
(?<start> (#{TAG_START}\s*)+)
(?<startInner> #{TAG_START}#{TAG_END}|#{TAG_EMPTY})?
(?<content> [^<]+)
(?<endInner> #{TAG_START}#{TAG_END}|#{TAG_EMPTY})?
(?<end> (\s*#{TAG_END})+)
/x
def process(source, scope)
@scope = scope
scan(source, :scope => scope, :strict => true) do |data|
add_translation data[:key], data[:value], data[:line_number]
end
end
def scan(source, options={})
options = {
:method => :scan
}.merge(options)
method = options[:method]
scope = options[:scope] ? options[:scope] + "." : ""
block_line_numbers = []
source.lines.each_with_index do |line, line_number|
line.scan(/#{I18N_CALL_START}.*(\}|$)/) do
block_line_numbers << line_number + 1
end
end
result = source.send(method, I18N_CALL) do
line_number = block_line_numbers.shift
match = Regexp.last_match
key = match[:key]
opts = match[:opts]
content = match[:content]
raise "invalid translation key #{key.inspect} on line #{line_number}" if options[:strict] && key !~ /\A#?[\w.]+\z/
key = scope + key if scope.size > 0 && !key.sub!(/\A#/, '')
convert_placeholders!(content, line_number)
wrappers = extract_wrappers!(content)
check_html(content, line_number) if options[:strict]
content.gsub!(/\s+/, ' ')
content.strip!
yield :key => key,
:value => content,
:options => opts,
:wrappers => wrappers,
:line_number => line_number
end
raise "possibly unterminated #t call (line #{block_line_numbers.shift} or earlier)" unless block_line_numbers.empty?
result
end
def convert_placeholders!(source, base_line_number)
source.lines.each_with_index do |line, line_number|
if line =~ /%h?\{(.*?)\}/
raise "use {{placeholder}} instead of %{placeholder}"
end
if line =~ /\{{2,3}(.*?)\}{2,3}/ && $1 =~ /[^a-z0-9_\.]/i
raise "helpers may not be used inside #t calls (line #{base_line_number + line_number})"
end
end
source.gsub!(/\{{3}(.*?)\}{3}/, '%h{\1}')
source.gsub!(/\{\{(.*?)\}\}/, '%{\1}')
end
def extract_wrappers!(source)
wrappers = {}
source.gsub!(I18N_WRAPPER) do
match = Regexp.last_match
if balanced_tags?(match[:start], match[:end])
value = "#{match[:start]}#{match[:startInner]}$1#{match[:endInner]}#{match[:end]}".gsub(/\s+/, ' ')
delimiter = wrappers[value] ||= '*' * (wrappers.size + 1)
"#{delimiter}#{match[:content]}#{delimiter}"
else
match.to_s
end
end
wrappers
end
def balanced_tags?(open, close)
open.scan(TAG_START).map { |tag| tag.match(TAG_NAME).to_s } ==
close.scan(TAG_END).map { |tag| tag.match(TAG_NAME).to_s }.reverse
end
def check_html(source, base_line_number)
source.lines.each_with_index do |line, line_number|
if line =~ /<[^>]+>/
raise "translation contains un-wrapper-ed markup (line #{base_line_number + line_number}). hint: use a placeholder, or balance your markup"
end
end
end
end
end

View File

@ -1,109 +0,0 @@
#
# Copyright (C) 2011 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require 'spec_helper'
module I18nExtraction
describe HandlebarsExtractor do
def extract(source, scope = 'asdf', options = {})
scope_results = scope && (options.has_key?(:scope_results) ? options.delete(:scope_results) : true)
extractor = HandlebarsExtractor.new
extractor.process(source, scope)
(scope_results ?
scope.split(/\./).inject(extractor.translations) { |hash, s| hash[s] } :
extractor.translations) || {}
end
context "keys" do
it "should allow valid string keys" do
extract('{{#t "foo"}}Foo{{/t}}').should eql({'foo' => "Foo"})
end
it "should disallow everything else" do
lambda { extract '{{#t "foo foo"}}Foo{{/t}}' }.should raise_error 'invalid translation key "foo foo" on line 1'
end
end
context "well-formed-ness" do
it "should make sure all #t calls are closed" do
lambda { extract "{{#t \"foo\"}}Foo{{/t}}\n{{#t \"bar\"}}...\nruh-roh\n" }.should raise_error /possibly unterminated #t call \(line 2/
end
end
context "values" do
it "should strip extraneous whitespace" do
extract("{{#t \"foo\"}}\t Foo\n foo\r\n\ffoo!!! {{/t}}").should eql({'foo' => 'Foo foo foo!!!'})
end
end
context "placeholders" do
it "should allow simple placeholders" do
extract('{{#t "foo"}}Hello {{user.name}}{{/t}}').should eql({'foo' => 'Hello %{user.name}'})
end
it "should disallow helpers or anything else" do
lambda { extract '{{#t "foo"}}Hello {{call a helper}}{{/t}}' }.should raise_error 'helpers may not be used inside #t calls (line 1)'
end
end
context "wrappers" do
it "should infer wrappers" do
extract('{{#t "foo"}}Be sure to <a href="{{url}}">log in</a>. <b>Don\'t</b> you <b>dare</b> forget!!!{{/t}}').should eql({'foo' => 'Be sure to *log in*. **Don\'t** you **dare** forget!!!'})
end
it "should not infer wrappers from unbalanced tags" do
lambda { extract '{{#t "foo"}}you are <b><i>so cool</i></strong>{{/t}}' }.should raise_error 'translation contains un-wrapper-ed markup (line 1). hint: use a placeholder, or balance your markup'
end
it "should allow empty tags on either side of the wrapper" do
extract('{{#t "bar"}}you can <button><i class="icon-email"></i>send an email</button>{{/t}}').should eql({'bar' => 'you can *send an email*'})
extract('{{#t "baz"}}this is <b>so cool!<img /></b>{{/t}}').should eql({'baz' => 'this is *so cool!*'})
end
it "should disallow any un-wrapper-ed html" do
lambda { extract '{{#t "foo"}}check out this pic: <img src="pic.gif">{{/t}}' }.should raise_error 'translation contains un-wrapper-ed markup (line 1). hint: use a placeholder, or balance your markup'
end
end
context "scoping" do
it "should auto-scope relative keys to the current scope" do
extract('{{#t "foo"}}Foo{{/t}}', 'asdf', :scope_results => false).should eql({'asdf' => {'foo' => "Foo"}})
end
it "should not auto-scope absolute keys" do
extract('{{#t "#foo"}}Foo{{/t}}', 'asdf', :scope_results => false).should eql({'foo' => "Foo"})
end
end
context "collisions" do
it "should not let you reuse a key" do
lambda { extract '{{#t "foo"}}Foo{{/t}}{{#t "foo"}}foo{{/t}}' }.should raise_error 'cannot reuse key "asdf.foo"'
end
it "should not let you use a scope as a key" do
lambda { extract '{{#t "foo.bar"}}bar{{/t}}{{#t "foo"}}foo{{/t}}' }.should raise_error '"asdf.foo" used as both a scope and a key'
end
it "should not let you use a key as a scope" do
lambda { extract '{{#t "foo"}}foo{{/t}}{{#t "foo.bar"}}bar{{/t}}' }.should raise_error '"asdf.foo" used as both a scope and a key'
end
end
end
end