translation checker/extractor for rb/erb files
Change-Id: I019f701758d35d630bf39141bb283911cc483aa5 Reviewed-on: https://gerrit.instructure.com/4237 Reviewed-by: Zach Wily <zach@instructure.com> Reviewed-by: Brian Palmer <brianp@instructure.com> Tested-by: Hudson <hudson@instructure.com>
This commit is contained in:
Jon Jensen
Zach Wily
parent
971979a7ce
commit
f544db6bc1
2
Gemfile
2
Gemfile
|
|
@ -69,6 +69,8 @@ end
|
|||
|
||||
group :development do
|
||||
gem 'ruby-debug', '0.10.4'
|
||||
gem 'ruby_parser', '2.0.6'
|
||||
gem 'sexp_processor', '3.0.5'
|
||||
end
|
||||
|
||||
group :redis do
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
class I18nExtractor < SexpProcessor
|
||||
attr_reader :translations, :total, :total_unique
|
||||
attr_accessor :scope
|
||||
|
||||
def initialize
|
||||
super
|
||||
@scope = ''
|
||||
@translations = {}
|
||||
@total = 0
|
||||
@total_unique = 0
|
||||
end
|
||||
|
||||
def process_defn(exp)
|
||||
exp.shift
|
||||
@current_defn = exp.shift
|
||||
process exp.shift until exp.empty?
|
||||
@current_defn = nil
|
||||
s
|
||||
end
|
||||
|
||||
TRANSLATE_CALLS = [:t, :ot, :mt, :translate, :before_label]
|
||||
LABEL_CALLS = [:label, :blabel]
|
||||
ALL_CALLS = TRANSLATE_CALLS + LABEL_CALLS + [:label_with_symbol_translation]
|
||||
|
||||
def process_call(exp)
|
||||
exp.shift
|
||||
receiver = process(exp.shift)
|
||||
method = exp.shift
|
||||
|
||||
call_scope = @scope
|
||||
if receiver && receiver.last == :Plugin && method == :register &&
|
||||
exp.first && exp.first.sexp_type == :arglist &&
|
||||
exp.first[1] && [:lit, :str].include?(exp.first[1].sexp_type)
|
||||
call_scope = "plugins.#{exp.first[1].last}."
|
||||
end
|
||||
|
||||
# ignore things like mt's t call
|
||||
unless ALL_CALLS.include?(@current_defn)
|
||||
if TRANSLATE_CALLS.include?(method)
|
||||
process_translate_call(receiver, method, exp.shift)
|
||||
elsif LABEL_CALLS.include?(method)
|
||||
process_label_call(receiver, method, exp.shift)
|
||||
end
|
||||
end
|
||||
with_scope(call_scope) do
|
||||
process exp.shift until exp.empty?
|
||||
end
|
||||
s
|
||||
end
|
||||
|
||||
def with_scope(scope)
|
||||
orig_scope = @scope
|
||||
@scope = scope
|
||||
yield if block_given?
|
||||
ensure
|
||||
@scope = orig_scope
|
||||
end
|
||||
|
||||
def process_translate_call(receiver, method, args)
|
||||
args.shift
|
||||
unless args.size >= 2
|
||||
if method == :before_label
|
||||
process args.shift until args.empty?
|
||||
return
|
||||
elsif receiver.last == :I18n && args.size == 1
|
||||
return
|
||||
else
|
||||
raise "insufficient arguments for translate call"
|
||||
end
|
||||
end
|
||||
|
||||
key = process_translation_key(receiver, args.shift, method == :before_label ? 'labels.' : '')
|
||||
|
||||
default = process_default_translation(args.shift, key)
|
||||
|
||||
options = if args.first.is_a?(Sexp)
|
||||
if args.first.sexp_type != :hash
|
||||
raise "translate options must be a hash: #{key.inspect}"
|
||||
end
|
||||
hash = args.shift
|
||||
hash.shift
|
||||
(0...(hash.size/2)).map{ |i|
|
||||
process hash[i * 2 + 1]
|
||||
raise "option keys must be strings or symbols" unless [:lit, :str].include?(hash[i * 2].sexp_type)
|
||||
hash[i * 2].last.to_sym
|
||||
}
|
||||
else
|
||||
[]
|
||||
end
|
||||
|
||||
# single word count/pluralization fu
|
||||
if default.is_a?(String) && default =~ /\A\w+\z/ && options.include?(:count)
|
||||
default = {:one => "1 #{default}", :other => "%{count} #{default.pluralize}"}
|
||||
end
|
||||
|
||||
(default.is_a?(String) ? {nil => default} : default).each_pair do |k, str|
|
||||
sub_key = k ? "#{key}.#{k}" : key
|
||||
str.scan(/%\{([^\}]+)\}/) do |match|
|
||||
unless options.include?(match[0].to_sym)
|
||||
raise "interpolation value not provided for #{match[0].to_sym.inspect} (#{sub_key.inspect})"
|
||||
end
|
||||
end
|
||||
add_translation sub_key, str
|
||||
end
|
||||
end
|
||||
|
||||
# stuff we want:
|
||||
# label :bar, :foo, :en => "Foo"
|
||||
# label :bar, :foo, :foo_key, :en => "Foo"
|
||||
# f.label :foo, :en => "Foo"
|
||||
# f.label :foo, :foo_key, :en => "Foo"
|
||||
def process_label_call(receiver, method, args)
|
||||
args.shift
|
||||
args.shift unless receiver # remove object_name arg
|
||||
|
||||
inferred = false
|
||||
default = nil
|
||||
key_arg = if args.size == 1 || args[1] && args[1].is_a?(Sexp) && args[1].sexp_type == :hash
|
||||
inferred = true
|
||||
args.shift
|
||||
elsif args[1].is_a?(Sexp)
|
||||
args.shift
|
||||
args.shift
|
||||
end
|
||||
if args.first.is_a?(Sexp) && args.first.sexp_type == :hash
|
||||
hash_args = args.first
|
||||
hash_args.shift
|
||||
(0...hash_args.size/2).each do |i|
|
||||
key = hash_args[2*i]
|
||||
value = hash_args[2*i + 1]
|
||||
if [:lit, :str].include?(key.sexp_type) && key.last.to_s == 'en'
|
||||
default = process_possible_string_concat(value)
|
||||
else
|
||||
process key
|
||||
process value
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if key_arg
|
||||
raise "invalid/missing en default #{args.first.inspect}" if (inferred || key_arg.sexp_type == :lit) && !default
|
||||
if default
|
||||
key = process_translation_key(receiver, key_arg, 'labels.', inferred)
|
||||
add_translation key, default
|
||||
else
|
||||
process key_arg
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def process_translation_key(receiver, exp, relative_scope, allow_strings=true)
|
||||
unless exp.is_a?(Sexp) && (exp.sexp_type == :lit || allow_strings && exp.sexp_type == :str)
|
||||
raise "invalid translation key #{exp.inspect}"
|
||||
end
|
||||
key = exp.pop.to_s
|
||||
if key =~ /\A#/
|
||||
key.sub!(/\A#/, '')
|
||||
else
|
||||
raise "ambiguous translation key #{key.inspect}" if @scope.empty? && receiver.nil?
|
||||
key = @scope + relative_scope + key
|
||||
end
|
||||
end
|
||||
|
||||
def process_default_translation(exp, key)
|
||||
raise "invalid en default #{exp.inspect}" unless exp.is_a?(Sexp)
|
||||
if exp.sexp_type == :hash
|
||||
exp.shift
|
||||
hash = Hash[*exp.map{ |e| process_possible_string_concat(e, :allow_symbols => true) }]
|
||||
if (hash.keys - [:one, :other, :zero]).size > 0
|
||||
raise "invalid :count sub-key(s): #{exp.inspect}"
|
||||
elsif hash.values.any?{ |v| !v.is_a?(String) }
|
||||
raise "invalid en count default(s): #{exp.inspect}"
|
||||
end
|
||||
hash
|
||||
else
|
||||
process_possible_string_concat(exp, :top_level_error => lambda{ |exp| "invalid en default #{exp.inspect}" })
|
||||
end
|
||||
rescue
|
||||
raise "#{$!} (#{key.inspect})"
|
||||
end
|
||||
|
||||
def process_possible_string_concat(exp, options={})
|
||||
if exp.sexp_type == :str
|
||||
exp.last
|
||||
elsif exp.sexp_type == :lit && options.delete(:allow_symbols)
|
||||
exp.last
|
||||
elsif exp.sexp_type == :call && exp[2] == :+ && exp.last && exp.last.sexp_type == :arglist && exp.last.size == 2 && exp.last.last.sexp_type == :str
|
||||
process_possible_string_concat(exp[1]) + exp.last.last.last
|
||||
else
|
||||
raise options[:top_level_error] ? options[:top_level_error].call(exp) : "unsupported string concatenation: #{exp.inspect}"
|
||||
end
|
||||
end
|
||||
|
||||
def add_translation(full_key, default)
|
||||
@total += 1
|
||||
scope = full_key.split('.')
|
||||
key = scope.pop
|
||||
hash = @translations
|
||||
while s = scope.shift
|
||||
if hash[s]
|
||||
raise "#{full_key.sub((scope.empty? ? '' : '.' + scope.join('.')) + '.' + key, '').inspect} used as both a scope and a key" unless hash[s].is_a?(Hash)
|
||||
else
|
||||
hash[s] = {}
|
||||
end
|
||||
hash = hash[s]
|
||||
end
|
||||
if hash[key]
|
||||
if hash[key] != default
|
||||
if hash[key].is_a?(Hash)
|
||||
raise "#{full_key.inspect} used as both a scope and a key"
|
||||
else
|
||||
raise "cannot reuse key #{full_key.inspect}"
|
||||
end
|
||||
end
|
||||
else
|
||||
@total_unique += 1
|
||||
hash[key] = default
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
namespace :i18n do
|
||||
desc "Verifies all translation calls"
|
||||
task :check => :environment do
|
||||
STI_SUPERCLASSES = (`grep '^class.*<' ./app/models/*rb|grep -v '::'|sed 's~.*< ~~'|sort|uniq`.split("\n") - ['OpenStruct', 'Tableless']).
|
||||
map{ |name| name.underscore + '.' }
|
||||
|
||||
def infer_scope(filename)
|
||||
case filename
|
||||
when /app\/controllers\//
|
||||
scope = filename.gsub(/.*app\/controllers\/|controller.rb/, '').gsub(/\/_?|_\z/, '.')
|
||||
scope == 'application.' ? '' : scope
|
||||
when /app\/messages\//
|
||||
filename.gsub(/.*app\/|erb/, '').gsub(/\/_?/, '.')
|
||||
when /app\/models\//
|
||||
scope = filename.gsub(/.*app\/models\/|rb/, '')
|
||||
STI_SUPERCLASSES.include?(scope) ? '' : scope
|
||||
when /app\/views\//
|
||||
filename.gsub(/.*app\/views\/|html\.erb/, '').gsub(/\/_?/, '.')
|
||||
else
|
||||
''
|
||||
end
|
||||
end
|
||||
|
||||
COLOR_ENABLED = ($stdout.tty? rescue false)
|
||||
def color(text, color_code)
|
||||
COLOR_ENABLED ? "#{color_code}#{text}\e[0m" : text
|
||||
end
|
||||
|
||||
def green(text)
|
||||
color(text, "\e[32m")
|
||||
end
|
||||
|
||||
def red(text)
|
||||
color(text, "\e[31m")
|
||||
end
|
||||
|
||||
files = Dir.glob('./**/*rb').
|
||||
reject{ |file| file =~ /\A\.\/(vendor\/plugins\/rails_xss|db|spec)\// }
|
||||
|
||||
t = Time.now
|
||||
@extractor = I18nExtractor.new
|
||||
@errors = []
|
||||
files.each do |file|
|
||||
begin
|
||||
source = File.read(file)
|
||||
source = Erubis::Eruby.new(source).src if file =~ /\.erb\z/
|
||||
|
||||
sexps = RubyParser.new.parse(source)
|
||||
@extractor.scope = infer_scope(file)
|
||||
@extractor.process(sexps)
|
||||
print green "."
|
||||
rescue SyntaxError, StandardError
|
||||
@errors << "#{$!}\n#{file}"
|
||||
print red "F"
|
||||
end
|
||||
end
|
||||
print "\n\n"
|
||||
failure = @errors.size > 0
|
||||
|
||||
@errors.each_index do |i|
|
||||
puts "#{i+1})"
|
||||
puts red @errors[i]
|
||||
print "\n"
|
||||
end
|
||||
|
||||
print "Finished in #{Time.now - t} seconds\n\n"
|
||||
puts send((failure ? :red : :green), "#{files.size} files, #{@extractor.total_unique} strings, #{@errors.size} failures")
|
||||
raise "check command encountered errors" if failure
|
||||
end
|
||||
|
||||
desc "Generates a new en.yml file for all translations"
|
||||
task :generate => :check do
|
||||
yaml_dir = './config/locales/generated'
|
||||
FileUtils.mkdir_p(File.join(yaml_dir))
|
||||
class Hash
|
||||
# for sorted goodness
|
||||
def to_yaml( opts = {} )
|
||||
YAML::quick_emit( object_id, opts ) do |out|
|
||||
out.map( taguri, to_yaml_style ) do |map|
|
||||
sort.each do |k, v|
|
||||
map.add( k, v )
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
yaml_file = File.join(yaml_dir, "en.yml")
|
||||
File.open(File.join(RAILS_ROOT, yaml_file), "w") do |file|
|
||||
file.write({'en' => @extractor.translations}.to_yaml)
|
||||
end
|
||||
print "Wrote new #{yaml_file}\n\n"
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
#
|
||||
# Copyright (C) 2011 Instructure, Inc.
|
||||
#
|
||||
# This file is part of Canvas.
|
||||
#
|
||||
# Canvas is free software: you can redistribute it and/or modify it under
|
||||
# the terms of the GNU Affero General Public License as published by the Free
|
||||
# Software Foundation, version 3 of the License.
|
||||
#
|
||||
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License along
|
||||
# with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
|
||||
|
||||
describe I18nExtractor do
|
||||
def extract(source, scope = 'asdf.', scope_results = true)
|
||||
sexps = RubyParser.new.parse(source)
|
||||
extractor = I18nExtractor.new
|
||||
extractor.scope = scope
|
||||
extractor.process(sexps)
|
||||
(scope_results ?
|
||||
scope.split(/\./).inject(extractor.translations) { |hash, s| hash[s] } :
|
||||
extractor.translations) || {}
|
||||
end
|
||||
|
||||
context "keys" do
|
||||
it "should allow string keys" do
|
||||
extract("t 'foo', \"Foo\"").should == {'foo' => "Foo"}
|
||||
end
|
||||
|
||||
it "should allow symbol keys" do
|
||||
extract("t :foo, \"Foo\"").should == {'foo' => "Foo"}
|
||||
end
|
||||
|
||||
it "should disallow everything else" do
|
||||
lambda{ extract "t foo, \"Foo\"" }.should raise_error /invalid translation key/
|
||||
lambda{ extract "t \"f\#{o}o\", \"Foo\"" }.should raise_error /invalid translation key/
|
||||
lambda{ extract "t true ? :foo : :bar, \"Foo\"" }.should raise_error /invalid translation key/
|
||||
end
|
||||
end
|
||||
|
||||
context "default translations" do
|
||||
it "should allow strings" do
|
||||
extract("t 'foo', \"Foo\"").should == {'foo' => "Foo"}
|
||||
extract("t 'foo2', <<-STR\nFoo\nSTR").should == {'foo2' => "Foo\n"}
|
||||
extract("t 'foo', 'F' + 'o' + 'o'").should == {'foo' => "Foo"}
|
||||
end
|
||||
|
||||
it "should disallow everything else" do
|
||||
lambda{ extract "t 'foo', \"F\#{o}o\"" }.should raise_error /invalid en default/
|
||||
lambda{ extract "t 'foo', foo" }.should raise_error /invalid en default/
|
||||
lambda{ extract "t 'foo', (true ? 'Foo' : 'Bar')" }.should raise_error /invalid en default/
|
||||
end
|
||||
end
|
||||
|
||||
context "placeholders" do
|
||||
it "should ensure all placeholders have corresponding options" do
|
||||
lambda{ extract "t 'foo', 'i have a %{foo}'" }.should raise_error(/interpolation value not provided for :foo/)
|
||||
end
|
||||
end
|
||||
|
||||
context "pluralization" do
|
||||
it "should auto-pluralize single words + :count" do
|
||||
extract("t 'foo', 'Foo', :count => 1").should == {'foo' => {'one' => "1 Foo", 'other' => "%{count} Foos"}}
|
||||
end
|
||||
|
||||
it "should not auto-pluralize other strings + :count" do
|
||||
extract("t 'foo', 'Foo foo', :count => 1").should == {'foo' => "Foo foo"}
|
||||
end
|
||||
|
||||
it "should ensure valid pluralization sub-keys" do
|
||||
lambda{ extract "t 'foo', {:invalid => '%{count} Foo'}, :count => 1" }.should raise_error(/invalid :count sub-key\(s\):/)
|
||||
end
|
||||
end
|
||||
|
||||
context "labels" do
|
||||
it "should interpret symbol names as the key" do
|
||||
extract("label :thing, :the_foo, :foo, :en => 'Foo'").should == {'labels' => {"foo" => "Foo"}}
|
||||
extract("f.label :the_foo, :foo, :en => 'Foo'").should == {'labels' => {"foo" => "Foo"}}
|
||||
end
|
||||
|
||||
it "should infer the key from the method if not provided" do
|
||||
extract("label :thing, :the_foo, :en => 'Foo'").should == {'labels' => {"the_foo" => "Foo"}}
|
||||
extract("f.label :the_foo, :en => 'Foo'").should == {'labels' => {"the_foo" => "Foo"}}
|
||||
end
|
||||
|
||||
it "should skip label calls with non-symbol keys (i.e. just a standard label)" do
|
||||
extract("label :thing, :the_foo, 'foo'").should == {}
|
||||
extract("f.label :the_foo, 'foo'").should == {}
|
||||
end
|
||||
|
||||
it "should complain if a label call has a non-symbol key and a default" do
|
||||
lambda{ extract "label :thing, :the_foo, 'foo', :en => 'Foo'" }.should raise_error /invalid translation key/
|
||||
lambda{ extract "f.label :the_foo, 'foo', :en => 'Foo'" }.should raise_error /invalid translation key/
|
||||
end
|
||||
|
||||
it "should not auto-scope absolute keys" do
|
||||
extract("label :thing, :the_foo, :'#foo', :en => 'Foo'", '').should == {"foo" => "Foo"}
|
||||
extract("f.label :the_foo, :'#foo', :en => 'Foo'", '').should == {"foo" => "Foo"}
|
||||
end
|
||||
|
||||
it "should complain if no default is provided" do
|
||||
lambda{ extract "label :thing, :the_foo, :foo" }.should raise_error 'invalid/missing en default nil'
|
||||
lambda{ extract "f.label :the_foo, :foo" }.should raise_error 'invalid/missing en default nil'
|
||||
end
|
||||
end
|
||||
|
||||
context "nesting" do
|
||||
it "should correctly evaluate i18n calls that are arguments to other (possibly) i18n calls" do
|
||||
extract("t :foo, 'Foo said \"%{bar}\"', :bar => t(:bar, 'Hello bar')").should == {'foo' => 'Foo said "%{bar}"', 'bar' => 'Hello bar'}
|
||||
extract("label :thing, :the_foo, t(:bar, 'Bar')").should == {'bar' => 'Bar'}
|
||||
extract("f.label :the_foo, t(:bar, 'Bar')").should == {'bar' => 'Bar'}
|
||||
end
|
||||
|
||||
it "should ignore i18n calls within i18n method definitions" do
|
||||
extract("def t(*args); other.t *args; end").should == {}
|
||||
end
|
||||
end
|
||||
|
||||
context "scoping" do
|
||||
it "should auto-scope relative keys to the current scope" do
|
||||
extract("t 'foo', 'Foo'", 'asdf.', false).should == {'asdf' => {'foo' => "Foo"}}
|
||||
end
|
||||
|
||||
it "should not auto-scope absolute keys" do
|
||||
extract("t '#foo', 'Foo'", 'asdf.', false).should == {'foo' => "Foo"}
|
||||
end
|
||||
|
||||
it "should auto-scope plugin registration" do
|
||||
extract("Canvas::Plugin.register('dim_dim', :web_conferencing, {:name => lambda{ t :name, \"DimDim\" }})", '', false).should ==
|
||||
{'plugins' => {'dim_dim' => {'name' => "DimDim"}}}
|
||||
end
|
||||
|
||||
it "should require explicit keys if there is no scope" do
|
||||
lambda{ extract("t 'foo', 'Foo'", '') }.should raise_error 'ambiguous translation key "foo"'
|
||||
end
|
||||
end
|
||||
|
||||
context "collisions" do
|
||||
it "should not let you reuse a key" do
|
||||
lambda{ extract "t 'foo', 'Foo'\nt 'foo', 'foo'" }.should raise_error 'cannot reuse key "asdf.foo"'
|
||||
end
|
||||
|
||||
it "should not let you use a scope as a key" do
|
||||
lambda{ extract "t 'foo.bar', 'bar'\nt 'foo', 'foo'" }.should raise_error '"asdf.foo" used as both a scope and a key'
|
||||
end
|
||||
|
||||
it "should not let you use a key as a scope" do
|
||||
lambda{ extract "t 'foo', 'foo'\nt 'foo.bar', 'bar'" }.should raise_error '"asdf.foo" used as both a scope and a key'
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue