ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

parallelize build tasks 

a faster rake js:generate, js:build, canvas:compile_assets, and guard

will use 'coffee' binary if installed

even if it doesn't use 'coffee' binary it will be
a lot faster

`time rake js:generate`
before               => real	0m29.960s
with 'coffee' binary => real	0m4.342s
without              => real	0m8.202s

test plan:
 * run bundle exec guard; ensure coffeescripts are compiled to the
   correct directories
 * run rake js:generate; ditto

Change-Id: I8fc4d4a415e5c77d1efa910c0922588d3095446b
Reviewed-on: https://gerrit.instructure.com/9989
Tested-by: Hudson <hudson@instructure.com>
Reviewed-by: Ryan Shaw <ryan@instructure.com>
Reviewed-by: Jacob Fugal <jacob@instructure.com>
This commit is contained in:
Ryan Shaw 2012-04-11 16:31:15 -06:00 committed by Cody Cutrer
parent 784b160c48
commit 25725a198e
8 changed files with 148 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@ -67,6 +67,7 @@ group :test do
gem 'coffee-script'
gem 'coffee-script-source', '1.1.2' #pinned so everyone's compiled output matches
gem 'bluecloth', '2.0.10' # for generating api docs
gem 'parallel', '0.5.16'
gem 'parallel_tests-instructure', '0.6.19'
gem 'mocha', '0.10.0'
gem 'rcov', '0.9.9'
@ -80,6 +81,7 @@ end
group :development do
gem 'coffee-script'
gem 'coffee-script-source', '1.1.2' #pinned so everyone's compiled output matches
gem 'parallel', '0.5.16'
gem 'ruby-debug', '0.10.4'
gem 'ruby_parser', '2.0.6'
gem 'sexp_processor', '3.0.5'

32
guard/coffeescript.rb
View File

@ -63,20 +63,34 @@ module Guard
Formatter.info(message, :reset => true)
end
require 'parallel'
require 'lib/canvas/coffee_script'
def compile_files(files, watchers, options)
errors = []
changed_files = []
directories = detect_nested_directories(watchers, files, options)
directories.each do |directory, scripts|
scripts.each do |file|
begin
content = compile(file, options)
changed_files << write_javascript_file(content, file, directory, options)
rescue Exception => e
error_message = file + ': ' + e.message.to_s
errors << error_message
Formatter.error(error_message)
if Canvas::CoffeeScript.coffee_script_binary_is_available?
Parallel.each(directories.map, :in_threads => Parallel.processor_count) do |(directory, scripts)|
FileUtils.mkdir_p(File.expand_path(directory)) if !File.directory?(directory) && !options[:noop]
system('coffee', '-c', '-o', directory, *scripts)
if $?.exitstatus != 0
Formatter.error("Unable to compile coffeescripts in #{directory}")
else
changed_files.concat(scripts.map { |script| File.join(directory, File.basename(script.gsub(/(js\.coffee|coffee)$/, 'js'))) })
end
end
else
directories.each do |directory, scripts|
Parallel.each(scripts, :in_threads => Parallel.processor_count) do |file|
begin
content = compile(file, options)
changed_files << write_javascript_file(content, file, directory, options)
rescue Exception => e
error_message = file + ': ' + e.message.to_s
errors << error_message
Formatter.error(error_message)
end
end
end
end

2
guard/jst.rb
View File

@ -49,7 +49,7 @@ module Guard
#
# Compiles templates from app/views/jst to public/javascripts/jst
def run_on_change(paths)
paths.each do |path|
Parallel.each(paths, :in_threads => Parallel.processor_count) do |path|
begin
puts "Compiling: #{path}"
Handlebars.compile_file path, 'app/views/jst', @options[:output]

18
lib/canvas/coffee_script.rb Normal file
View File

@ -0,0 +1,18 @@
module Canvas
class CoffeeScript
def self.coffee_script_binary_is_available?
return @is_available if instance_variable_defined?(:@is_available)
coffee_is_installed = `which coffee` && $?.success?
if coffee_is_installed
coffee_version = `coffee -v`.strip
coffee_is_correct_version = coffee_version.match(::CoffeeScript.version)
unless coffee_is_correct_version
puts "--> WARNING #{coffee_version} != pinned coffee-script-source: #{::CoffeeScript.version}"
end
end
@is_available = coffee_is_installed && coffee_is_correct_version
end
end
end

17
lib/handlebars/handlebars.rb
View File

@ -17,9 +17,20 @@ class Handlebars
# plugin (string) - Optional plugin to which the files belong. Will be
# used in scoping the generated module names. If absent, but a file is
# visibly under a plugin, the plugin for that file will be inferred.
def compile(root_path, compiled_path, plugin=nil)
files = Dir["#{root_path}/**/**.handlebars"]
files.each { |file| compile_file file, root_path, compiled_path, plugin }
#
# OR an array of such
def compile(*args)
require 'parallel'
unless args.first.is_a? Array
args = [args]
end
files = []
args.each do |(root_path, compiled_path, plugin)|
files.concat(Dir["#{root_path}/**/**.handlebars"].map { |file| [file, root_path, compiled_path, plugin] })
end
Parallel.each(files, :in_threads => Parallel.processor_count) do |file|
compile_file *file
end
end
# Compiles a single file into a destination directory.

37
lib/tasks/canvas.rake
View File

@ -116,24 +116,35 @@ namespace :canvas do
desc "Compile javascript and css assets."
task :compile_assets do
puts "--> Compiling static assets [css]"
Rake::Task['css:generate'].invoke
threads = []
threads << Thread.new do
puts "--> Compiling static assets [css]"
Rake::Task['css:generate'].invoke
puts "--> Compiling static assets [jammit]"
output = `bundle exec jammit 2>&1`
raise "Error running jammit: \n#{output}\nABORTING" if $?.exitstatus != 0
puts "--> Compiling static assets [jammit]"
output = `bundle exec jammit 2>&1`
raise "Error running jammit: \n#{output}\nABORTING" if $?.exitstatus != 0
puts "--> Compiling static assets [javascript]"
Rake::Task['js:generate'].invoke
puts "--> Compiled static assets [css/jammit]"
end
puts "--> Generating js localization bundles"
Rake::Task['i18n:generate_js'].invoke
threads << Thread.new do
puts "--> Compiling static assets [javascript]"
Rake::Task['js:generate'].invoke
puts "--> Optimizing JavaScript [r.js]"
Rake::Task['js:build'].invoke
puts "--> Generating js localization bundles"
Rake::Task['i18n:generate_js'].invoke
puts "--> Generating documentation [yardoc]"
Rake::Task['doc:api'].invoke
puts "--> Optimizing JavaScript [r.js]"
Rake::Task['js:build'].invoke
end
threads << Thread.new do
puts "--> Generating documentation [yardoc]"
Rake::Task['doc:api'].invoke
end
threads.each(&:join)
end
end

91
lib/tasks/js.rake
View File

@ -15,11 +15,14 @@ namespace :js do
raise "PhantomJS tests failed" if exit_status != 0
end
def compile_coffescript(coffee_file)
destination = coffee_file.sub('app/coffeescripts', 'public/javascripts/compiled').
sub('spec/coffeescripts', 'spec/javascripts').
sub(%r{/javascripts/compiled/plugins/([^/]+)/}, '/plugins/\\1/javascripts/compiled/').
sub(%r{\.coffee$}, '.js')
def coffee_destination(dir_or_file)
dir_or_file.sub('app/coffeescripts', 'public/javascripts/compiled').
sub('spec/coffeescripts', 'spec/javascripts').
sub(%r{/javascripts/compiled/plugins/([^/]+)/}, '/plugins/\\1/javascripts/compiled/')
end
def compile_coffeescript(coffee_file)
destination = coffee_destination(coffee_file).sub(%r{\.coffee$}, '.js')
FileUtils.mkdir_p(File.dirname(destination))
File.open(destination, 'wb') do |out|
out.write CoffeeScript.compile(File.open(coffee_file))
@ -29,45 +32,75 @@ namespace :js do
desc "generates compiled coffeescript and handlebars templates"
task :generate do
require 'config/initializers/plugin_symlinks'
require 'coffee-script'
require 'fileutils'
require 'canvas'
require 'canvas/coffee_script'
# clear out all the files in case there are any old compiled versions of
# files that don't map to any source file anymore
FileUtils.rm_rf('public/javascripts/compiled')
FileUtils.rm_rf('public/javascripts/jst')
Dir.glob('spec/javascripts/**/*Spec.js') do |compiled_spec|
FileUtils.rm_f(compiled_spec)
end
Dir.glob('public/plugins/*/javascripts') do |plugin_dir|
FileUtils.rm_rf(plugin_dir + '/compiled')
FileUtils.rm_rf(plugin_dir + '/javascripts/jst')
paths_to_remove = [
'public/javascripts/compiled',
'public/javascripts/jst',
'public/plugins/*/javascripts/{compiled,javascripts/jst}'
] + Dir.glob('spec/javascripts/**/*Spec.js')
FileUtils.rm_rf(paths_to_remove)
threads = []
threads << Thread.new do
puts "--> Pre-compiling handlebars templates"
handlebars_time = Benchmark.realtime { Rake::Task['jst:compile'].invoke }
puts "--> Pre-compiling handlebars templates finished in #{handlebars_time}"
end
puts "--> Pre-compiling all handlebars templates"
Rake::Task['jst:compile'].invoke
puts "--> Compiling all Coffeescript"
Dir[Rails.root+'spec/coffeescripts/{,plugins/*/}**/*.coffee'].each do |file|
compile_coffescript file
end
Dir[Rails.root+'app/coffeescripts/{,plugins/*/}**/*.coffee'].each do |file|
compile_coffescript file
threads << Thread.new do
coffee_time = Benchmark.realtime do
require 'coffee-script'
if Canvas::CoffeeScript.coffee_script_binary_is_available?
puts "--> Compiling CoffeeScript with 'coffee' binary"
dirs = Dir[Rails.root+'{app,spec}/coffeescripts/{,plugins/*/}**/*.coffee'].
map { |f| File.dirname(f) }.uniq
Parallel.each(dirs, :in_threads => Parallel.processor_count) do |dir|
destination = coffee_destination(dir)
FileUtils.mkdir_p(destination)
system("coffee -c -o #{destination} #{dir}/*.coffee")
raise "Unable to compile coffeescripts in #{dir}" if $?.exitstatus != 0
end
else
puts "--> Compiling CoffeeScript with coffee-script gem"
files = Dir[Rails.root+'{app,spec}/coffeescripts/{,plugins/*/}**/*.coffee']
Parallel.each(files, :in_threads => Parallel.processor_count) do |file|
compile_coffeescript file
end
end
end
puts "--> Compiling CoffeeScript finished in #{coffee_time}"
end
threads.each(&:join)
end
desc "optimize and build js for production"
task :build do
require 'config/initializers/plugin_symlinks'
require 'parallel'
puts "--> Optimizing canvas-lms"
output = `node #{Rails.root}/node_modules/requirejs/bin/r.js -o #{Rails.root}/config/build.js 2>&1`
raise "Error running js:build: \n#{output}\nABORTING" if $?.exitstatus != 0
commands = []
commands << ['canvas-lms', "node #{Rails.root}/node_modules/requirejs/bin/r.js -o #{Rails.root}/config/build.js 2>&1"]
Dir[Rails.root+'vendor/plugins/*/config/build.js'].each do |buildfile|
files = Dir[Rails.root+'vendor/plugins/*/config/build.js']
files.each do |buildfile|
plugin = buildfile.gsub(%r{.*/vendor/plugins/(.*)/config/build\.js}, '\\1')
puts "--> Optimizing #{plugin} plugin"
output = `node #{Rails.root}/node_modules/requirejs/bin/r.js -o #{buildfile} 2>&1`
raise "Error running js:build: \n#{output}\nABORTING" if $?.exitstatus != 0
commands << ["#{plugin} plugin", "node #{Rails.root}/node_modules/requirejs/bin/r.js -o #{buildfile} 2>&1"]
end
Parallel.each(commands, :in_threads => Parallel.processor_count) do |(plugin, command)|
puts "--> Optimizing #{plugin}"
optimize_time = Benchmark.realtime do
output = `#{command}`
raise "Error running js:build: \n#{output}\nABORTING" if $?.exitstatus != 0
end
puts "--> Optimized #{plugin} in #{optimize_time}"
end
end

6
lib/tasks/jst.rake
View File

@ -5,12 +5,14 @@ namespace :jst do
task :compile do
require 'config/initializers/plugin_symlinks'
Handlebars.compile 'app/views/jst', 'public/javascripts/jst'
all_paths = []
all_paths << ['app/views/jst', 'public/javascripts/jst']
Dir[Rails.root+'app/views/jst/plugins/*'].each do |input_path|
plugin = input_path.sub(%r{.*app/views/jst/plugins/}, '')
output_path = "public/plugins/#{plugin}/javascripts/jst"
Handlebars.compile input_path, output_path, plugin
all_paths << [input_path, output_path, plugin]
end
Handlebars.compile *all_paths
end
end