translation checker/extractor for rb/erb files

Change-Id: I019f701758d35d630bf39141bb283911cc483aa5
Reviewed-on: https://gerrit.instructure.com/4237
Reviewed-by: Zach Wily <zach@instructure.com>
Reviewed-by: Brian Palmer <brianp@instructure.com>
Tested-by: Hudson <hudson@instructure.com>

Gemfile

@@ -69,6 +69,8 @@ end
 
 group :development do
   gem 'ruby-debug',   '0.10.4'
+  gem 'ruby_parser', '2.0.6'
+  gem 'sexp_processor', '3.0.5'
 end
 
 group :redis do

lib/i18n_extractor.rb

@@ -0,0 +1,220 @@
+class I18nExtractor < SexpProcessor
+  attr_reader :translations, :total, :total_unique
+  attr_accessor :scope
+
+  def initialize
+    super
+    @scope = ''
+    @translations = {}
+    @total = 0
+    @total_unique = 0
+  end
+
+  def process_defn(exp)
+    exp.shift
+    @current_defn = exp.shift
+    process exp.shift until exp.empty?
+    @current_defn = nil
+    s
+  end
+
+  TRANSLATE_CALLS = [:t, :ot, :mt, :translate, :before_label]
+  LABEL_CALLS = [:label, :blabel]
+  ALL_CALLS = TRANSLATE_CALLS + LABEL_CALLS + [:label_with_symbol_translation]
+
+  def process_call(exp)
+    exp.shift
+    receiver = process(exp.shift)
+    method = exp.shift
+
+    call_scope = @scope
+    if receiver && receiver.last == :Plugin && method == :register &&
+        exp.first && exp.first.sexp_type == :arglist &&
+        exp.first[1] && [:lit, :str].include?(exp.first[1].sexp_type)
+      call_scope = "plugins.#{exp.first[1].last}."
+    end
+
+    # ignore things like mt's t call
+    unless ALL_CALLS.include?(@current_defn)
+      if TRANSLATE_CALLS.include?(method)
+        process_translate_call(receiver, method, exp.shift)
+      elsif LABEL_CALLS.include?(method)
+        process_label_call(receiver, method, exp.shift)
+      end
+    end
+    with_scope(call_scope) do
+      process exp.shift until exp.empty?
+    end
+    s
+  end
+
+  def with_scope(scope)
+    orig_scope = @scope
+    @scope = scope
+    yield if block_given?
+  ensure
+    @scope = orig_scope
+  end
+
+  def process_translate_call(receiver, method, args)
+    args.shift
+    unless args.size >= 2
+      if method == :before_label
+        process args.shift until args.empty?
+        return
+      elsif receiver.last == :I18n && args.size == 1
+        return
+      else
+        raise "insufficient arguments for translate call"
+      end
+    end
+
+    key = process_translation_key(receiver, args.shift, method == :before_label ? 'labels.' : '')
+
+    default = process_default_translation(args.shift, key)
+
+    options = if args.first.is_a?(Sexp)
+      if args.first.sexp_type != :hash
+        raise "translate options must be a hash: #{key.inspect}"
+      end
+      hash = args.shift
+      hash.shift
+      (0...(hash.size/2)).map{ |i|
+        process hash[i * 2 + 1]
+        raise "option keys must be strings or symbols" unless [:lit, :str].include?(hash[i * 2].sexp_type)
+        hash[i * 2].last.to_sym
+      }
+    else
+      []
+    end
+
+    # single word count/pluralization fu
+    if default.is_a?(String) && default =~ /\A\w+\z/ && options.include?(:count)
+      default = {:one => "1 #{default}", :other => "%{count} #{default.pluralize}"}
+    end
+
+    (default.is_a?(String) ? {nil => default} : default).each_pair do |k, str|
+      sub_key = k ? "#{key}.#{k}" : key
+      str.scan(/%\{([^\}]+)\}/) do |match|
+        unless options.include?(match[0].to_sym)
+          raise "interpolation value not provided for #{match[0].to_sym.inspect} (#{sub_key.inspect})"
+        end
+      end
+      add_translation sub_key, str
+    end
+  end
+
+  # stuff we want:
+  #  label :bar, :foo, :en => "Foo"
+  #  label :bar, :foo, :foo_key, :en => "Foo"
+  #  f.label :foo, :en => "Foo"
+  #  f.label :foo, :foo_key, :en => "Foo"
+  def process_label_call(receiver, method, args)
+    args.shift
+    args.shift unless receiver # remove object_name arg
+
+    inferred = false
+    default = nil
+    key_arg = if args.size == 1 || args[1] && args[1].is_a?(Sexp) && args[1].sexp_type == :hash
+      inferred = true
+      args.shift
+    elsif args[1].is_a?(Sexp)
+      args.shift
+      args.shift
+    end
+    if args.first.is_a?(Sexp) && args.first.sexp_type == :hash
+      hash_args = args.first
+      hash_args.shift
+      (0...hash_args.size/2).each do |i|
+        key = hash_args[2*i]
+        value = hash_args[2*i + 1]
+        if [:lit, :str].include?(key.sexp_type) && key.last.to_s == 'en'
+          default = process_possible_string_concat(value)
+        else
+          process key
+          process value
+        end
+      end
+    end
+    
+    if key_arg
+      raise "invalid/missing en default #{args.first.inspect}" if (inferred || key_arg.sexp_type == :lit) && !default
+      if default
+        key = process_translation_key(receiver, key_arg, 'labels.', inferred)
+        add_translation key, default
+      else
+        process key_arg
+      end
+    end
+  end
+
+  def process_translation_key(receiver, exp, relative_scope, allow_strings=true)
+    unless exp.is_a?(Sexp) && (exp.sexp_type == :lit || allow_strings && exp.sexp_type == :str)
+      raise "invalid translation key #{exp.inspect}"
+    end
+    key = exp.pop.to_s
+    if key =~ /\A#/
+      key.sub!(/\A#/, '')
+    else
+      raise "ambiguous translation key #{key.inspect}" if @scope.empty? && receiver.nil?
+      key = @scope + relative_scope + key
+    end
+  end
+
+  def process_default_translation(exp, key)
+    raise "invalid en default #{exp.inspect}" unless exp.is_a?(Sexp)
+    if exp.sexp_type == :hash
+      exp.shift
+      hash = Hash[*exp.map{ |e| process_possible_string_concat(e, :allow_symbols => true) }]
+      if (hash.keys - [:one, :other, :zero]).size > 0
+        raise "invalid :count sub-key(s): #{exp.inspect}"
+      elsif hash.values.any?{ |v| !v.is_a?(String) }
+        raise "invalid en count default(s): #{exp.inspect}"
+      end
+      hash
+    else
+      process_possible_string_concat(exp, :top_level_error => lambda{ |exp| "invalid en default #{exp.inspect}" })
+    end
+  rescue
+    raise "#{$!} (#{key.inspect})"
+  end
+
+  def process_possible_string_concat(exp, options={})
+    if exp.sexp_type == :str
+      exp.last
+    elsif exp.sexp_type == :lit && options.delete(:allow_symbols)
+      exp.last
+    elsif exp.sexp_type == :call && exp[2] == :+ && exp.last && exp.last.sexp_type == :arglist && exp.last.size == 2 && exp.last.last.sexp_type == :str
+      process_possible_string_concat(exp[1]) + exp.last.last.last
+    else
+      raise options[:top_level_error] ? options[:top_level_error].call(exp) : "unsupported string concatenation: #{exp.inspect}"
+    end
+  end
+
+  def add_translation(full_key, default)
+    @total += 1
+    scope = full_key.split('.')
+    key = scope.pop
+    hash = @translations
+    while s = scope.shift
+      if hash[s]
+        raise "#{full_key.sub((scope.empty? ? '' : '.' + scope.join('.')) + '.' + key, '').inspect} used as both a scope and a key" unless hash[s].is_a?(Hash)
+      else
+        hash[s] = {}
+      end
+      hash = hash[s]
+    end
+    if hash[key]
+      if hash[key] != default
+        if hash[key].is_a?(Hash)
+          raise "#{full_key.inspect} used as both a scope and a key"
+        else
+          raise "cannot reuse key #{full_key.inspect}"
+        end
+      end
+    else
+      @total_unique += 1
+      hash[key] = default
+    end
+  end
+end

lib/tasks/i18n.rake

@@ -0,0 +1,93 @@
+namespace :i18n do
+  desc "Verifies all translation calls"
+  task :check => :environment do
+    STI_SUPERCLASSES = (`grep '^class.*<' ./app/models/*rb|grep -v '::'|sed 's~.*< ~~'|sort|uniq`.split("\n") - ['OpenStruct', 'Tableless']).
+      map{ |name| name.underscore + '.' }
+
+    def infer_scope(filename)
+      case filename
+        when /app\/controllers\//
+          scope = filename.gsub(/.*app\/controllers\/|controller.rb/, '').gsub(/\/_?|_\z/, '.')
+          scope == 'application.' ? '' : scope
+        when /app\/messages\//
+          filename.gsub(/.*app\/|erb/, '').gsub(/\/_?/, '.')
+        when /app\/models\//
+          scope = filename.gsub(/.*app\/models\/|rb/, '')
+          STI_SUPERCLASSES.include?(scope) ? '' : scope
+        when /app\/views\//
+          filename.gsub(/.*app\/views\/|html\.erb/, '').gsub(/\/_?/, '.')
+        else
+          ''
+      end
+    end
+
+    COLOR_ENABLED = ($stdout.tty? rescue false)
+    def color(text, color_code)
+      COLOR_ENABLED ? "#{color_code}#{text}\e[0m" : text
+    end
+
+    def green(text)
+      color(text, "\e[32m")
+    end
+
+    def red(text)
+      color(text, "\e[31m")
+    end
+
+    files = Dir.glob('./**/*rb').
+      reject{ |file| file =~ /\A\.\/(vendor\/plugins\/rails_xss|db|spec)\// }
+
+    t = Time.now
+    @extractor = I18nExtractor.new
+    @errors = []
+    files.each do |file|
+      begin
+        source = File.read(file)
+        source = Erubis::Eruby.new(source).src if file =~ /\.erb\z/
+
+        sexps = RubyParser.new.parse(source)
+        @extractor.scope = infer_scope(file)
+        @extractor.process(sexps)
+        print green "."
+      rescue SyntaxError, StandardError
+        @errors << "#{$!}\n#{file}"
+        print red "F"
+      end
+    end
+    print "\n\n"
+    failure = @errors.size > 0
+
+    @errors.each_index do |i|
+      puts "#{i+1})"
+      puts red @errors[i]
+      print "\n"
+    end
+
+    print "Finished in #{Time.now - t} seconds\n\n"
+    puts send((failure ? :red : :green), "#{files.size} files, #{@extractor.total_unique} strings, #{@errors.size} failures")
+    raise "check command encountered errors" if failure
+  end
+
+  desc "Generates a new en.yml file for all translations"
+  task :generate => :check do
+    yaml_dir = './config/locales/generated'
+    FileUtils.mkdir_p(File.join(yaml_dir))
+    class Hash
+      # for sorted goodness
+      def to_yaml( opts = {} )
+        YAML::quick_emit( object_id, opts ) do |out|
+          out.map( taguri, to_yaml_style ) do |map|
+            sort.each do |k, v|
+              map.add( k, v )
+            end
+          end
+        end
+      end
+    end
+    yaml_file = File.join(yaml_dir, "en.yml")
+    File.open(File.join(RAILS_ROOT, yaml_file), "w") do |file|
+      file.write({'en' => @extractor.translations}.to_yaml)
+    end
+    print "Wrote new #{yaml_file}\n\n"
+  end
+end

spec/lib/i18n_extractor_spec.rb

@@ -0,0 +1,158 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+
+describe I18nExtractor do
+  def extract(source, scope = 'asdf.', scope_results = true)
+    sexps = RubyParser.new.parse(source)
+    extractor = I18nExtractor.new
+    extractor.scope = scope
+    extractor.process(sexps)
+    (scope_results ?
+      scope.split(/\./).inject(extractor.translations) { |hash, s| hash[s] } :
+      extractor.translations) || {}
+  end
+
+  context "keys" do
+    it "should allow string keys" do
+      extract("t 'foo', \"Foo\"").should == {'foo' => "Foo"}
+    end
+
+    it "should allow symbol keys" do
+      extract("t :foo, \"Foo\"").should == {'foo' => "Foo"}
+    end
+
+    it "should disallow everything else" do
+      lambda{ extract "t foo, \"Foo\"" }.should raise_error /invalid translation key/
+      lambda{ extract "t \"f\#{o}o\", \"Foo\"" }.should raise_error /invalid translation key/
+      lambda{ extract "t true ? :foo : :bar, \"Foo\"" }.should raise_error /invalid translation key/
+    end
+  end
+
+  context "default translations" do
+    it "should allow strings" do
+      extract("t 'foo', \"Foo\"").should == {'foo' => "Foo"}
+      extract("t 'foo2', <<-STR\nFoo\nSTR").should == {'foo2' => "Foo\n"}
+      extract("t 'foo', 'F' + 'o' + 'o'").should == {'foo' => "Foo"}
+    end
+
+    it "should disallow everything else" do
+      lambda{ extract "t 'foo', \"F\#{o}o\"" }.should raise_error /invalid en default/
+      lambda{ extract "t 'foo', foo" }.should raise_error /invalid en default/
+      lambda{ extract "t 'foo', (true ? 'Foo' : 'Bar')" }.should raise_error /invalid en default/
+    end
+  end
+
+  context "placeholders" do
+    it "should ensure all placeholders have corresponding options" do
+      lambda{ extract "t 'foo', 'i have a %{foo}'" }.should raise_error(/interpolation value not provided for :foo/)
+    end
+  end
+
+  context "pluralization" do
+    it "should auto-pluralize single words + :count" do
+      extract("t 'foo', 'Foo', :count => 1").should == {'foo' => {'one' => "1 Foo", 'other' => "%{count} Foos"}}
+    end
+
+    it "should not auto-pluralize other strings + :count" do
+      extract("t 'foo', 'Foo foo', :count => 1").should == {'foo' => "Foo foo"}
+    end
+
+    it "should ensure valid pluralization sub-keys" do
+      lambda{ extract "t 'foo', {:invalid => '%{count} Foo'}, :count => 1" }.should raise_error(/invalid :count sub-key\(s\):/)
+    end
+  end
+
+  context "labels" do
+    it "should interpret symbol names as the key" do
+      extract("label :thing, :the_foo, :foo, :en => 'Foo'").should == {'labels' => {"foo" => "Foo"}}
+      extract("f.label :the_foo, :foo, :en => 'Foo'").should == {'labels' => {"foo" => "Foo"}}
+    end
+
+    it "should infer the key from the method if not provided" do
+      extract("label :thing, :the_foo, :en => 'Foo'").should == {'labels' => {"the_foo" => "Foo"}}
+      extract("f.label :the_foo, :en => 'Foo'").should == {'labels' => {"the_foo" => "Foo"}}
+    end
+
+    it "should skip label calls with non-symbol keys (i.e. just a standard label)" do
+      extract("label :thing, :the_foo, 'foo'").should == {}
+      extract("f.label :the_foo, 'foo'").should == {}
+    end
+
+    it "should complain if a label call has a non-symbol key and a default" do
+      lambda{ extract "label :thing, :the_foo, 'foo', :en => 'Foo'" }.should raise_error /invalid translation key/
+      lambda{ extract "f.label :the_foo, 'foo', :en => 'Foo'" }.should raise_error /invalid translation key/
+    end
+
+    it "should not auto-scope absolute keys" do
+      extract("label :thing, :the_foo, :'#foo', :en => 'Foo'", '').should == {"foo" => "Foo"}
+      extract("f.label :the_foo, :'#foo', :en => 'Foo'", '').should == {"foo" => "Foo"}
+    end
+
+    it "should complain if no default is provided" do
+      lambda{ extract "label :thing, :the_foo, :foo" }.should raise_error 'invalid/missing en default nil'
+      lambda{ extract "f.label :the_foo, :foo" }.should raise_error 'invalid/missing en default nil'
+    end
+  end
+
+  context "nesting" do
+    it "should correctly evaluate i18n calls that are arguments to other (possibly) i18n calls" do
+      extract("t :foo, 'Foo said \"%{bar}\"', :bar => t(:bar, 'Hello bar')").should == {'foo' => 'Foo said "%{bar}"', 'bar' => 'Hello bar'}
+      extract("label :thing, :the_foo, t(:bar, 'Bar')").should == {'bar' => 'Bar'}
+      extract("f.label :the_foo, t(:bar, 'Bar')").should == {'bar' => 'Bar'}
+    end
+
+    it "should ignore i18n calls within i18n method definitions" do
+      extract("def t(*args); other.t *args; end").should == {}
+    end
+  end
+
+  context "scoping" do
+    it "should auto-scope relative keys to the current scope" do
+      extract("t 'foo', 'Foo'", 'asdf.', false).should == {'asdf' => {'foo' => "Foo"}}
+    end
+
+    it "should not auto-scope absolute keys" do
+      extract("t '#foo', 'Foo'", 'asdf.', false).should == {'foo' => "Foo"}
+    end
+
+    it "should auto-scope plugin registration" do
+      extract("Canvas::Plugin.register('dim_dim', :web_conferencing, {:name => lambda{ t :name, \"DimDim\" }})", '', false).should ==
+        {'plugins' => {'dim_dim' => {'name' => "DimDim"}}}
+    end
+
+    it "should require explicit keys if there is no scope" do
+      lambda{ extract("t 'foo', 'Foo'", '') }.should raise_error 'ambiguous translation key "foo"'
+    end
+  end
+
+  context "collisions" do
+    it "should not let you reuse a key" do
+      lambda{ extract "t 'foo', 'Foo'\nt 'foo', 'foo'" }.should raise_error 'cannot reuse key "asdf.foo"'
+    end
+
+    it "should not let you use a scope as a key" do
+      lambda{ extract "t 'foo.bar', 'bar'\nt 'foo', 'foo'" }.should raise_error '"asdf.foo" used as both a scope and a key'
+    end
+
+    it "should not let you use a key as a scope" do
+      lambda{ extract "t 'foo', 'foo'\nt 'foo.bar', 'bar'" }.should raise_error '"asdf.foo" used as both a scope and a key'
+    end
+  end
+end