make rake canvas:compile_assets a TON faster

...by parallelizing everything I could

fixes: CNVS-12362

`time bundle exec rake canvas:compile_assets`
(on my I7 2.3GHz macbook pro)
before: "322.56s user 26.62s system 173% cpu 3:21.05 total"
after: "425.91s user 30.71s system 413% cpu 1:50.35 total"

If you ever find yourself needing to run everything all
sequentially (and not in multiple threads), you can
set the environment variable CANVAS_BUILD_CONCURRENCY=1
by default it will use all the cores your system has

You're welcome, anyone that has ever had to build canvas
or wait for jenkins to run :)

Test Plan:
* run "time bundle exec rake canvas:compile_assets"
* checkout this code
* run it again
* verify that it is faster, and everything works

Change-Id: Ib01506bc9638e86f4a329284458706279ef751ab
Reviewed-on: https://gerrit.instructure.com/33127
Tested-by: Jenkins <jenkins@instructure.com>
Reviewed-by: Jeremy Stanley <jeremy@instructure.com>
QA-Review: Nathan Rogowski <nathan@instructure.com>
Product-Review: Bracken Mosbacker <bracken@instructure.com>

lib/tasks/canvas.rake

@@ -2,6 +2,13 @@ $canvas_tasks_loaded ||= false
 unless $canvas_tasks_loaded
 $canvas_tasks_loaded = true
 
+def log_time(name, &block)
+  puts "--> Starting: '#{name}'"
+  time = Benchmark.realtime(&block)
+  puts "--> Finished: '#{name}' in #{time}"
+  time
+end
+
 def check_syntax(files)
   quick = ENV["quick"] && ENV["quick"] == "true"
   puts "--> Checking Syntax...."
@@ -103,28 +110,46 @@ namespace :canvas do
     generate_docs = args[:generate_documentation]
     generate_docs = 'true' if !['true', 'false'].include?(args[:generate_documentation])
 
-    puts "--> Compiling static assets [css]"
-    Rake::Task['css:generate'].invoke
-    Rake::Task['css:styleguide'].invoke
+    tasks = {
+      "Compile sass and make jammit css bundles" => -> {
+        log_time('css:generate') do
+          Rake::Task['css:generate'].invoke
+        end
 
-    puts "--> Compiling static assets [jammit]"
-    output = `bundle exec jammit 2>&1`
-    raise "Error running jammit: \n#{output}\nABORTING" if $?.exitstatus != 0
-    puts "--> Compiled static assets [css/jammit]"
-
-    puts "--> Compiling static assets [javascript]"
-    Rake::Task['js:generate'].invoke
-
-    puts "--> Generating js localization bundles"
-    Rake::Task['i18n:generate_js'].invoke
-
-    puts "--> Optimizing JavaScript [r.js]"
-    Rake::Task['js:build'].invoke
+        log_time("Jammit") do
+          require 'jammit'
+          Jammit.package!
+        end
+      },
+      "css:styleguide" => -> {
+        Rake::Task['css:styleguide'].invoke
+      },
+      "compile coffee, js 18n, and run r.js optimizer" => -> {
+        ['js:generate', 'i18n:generate_js', 'js:build'].each do |name|
+          log_time(name) { Rake::Task[name].invoke }
+        end
+      }
+    }
 
     if generate_docs == 'true'
-      puts "--> Generating documentation [yardoc]"
-      Rake::Task['doc:api'].invoke
+      tasks["Generate documentation [yardoc]"] = -> {
+        Rake::Task['doc:api'].invoke
+      }
+    end
+
+
+
+    require 'parallel'
+    processes = ENV['CANVAS_BUILD_CONCURRENCY'] || Parallel.processor_count
+    puts "working in #{processes} processes"
+    times = nil
+    real_time = Benchmark.realtime do
+      times = Parallel.map(tasks, :in_processes => processes.to_i) do |name, lamduh|
+        log_time(name) { lamduh.call }
       end
+    end
+    combined_time = times.reduce(:+)
+    puts "Finished compiling assets in #{real_time}. parralellisim saved #{combined_time - real_time} (#{real_time.to_f / combined_time.to_f * 100.0}%)"
   end
 
   desc "Check static assets and generate api documentation."
@@ -227,4 +252,4 @@ namespace :db do
   end
 end
 
-end
+end

package.json

@@ -18,7 +18,8 @@
     "karma-ie-launcher": "~0.1.1",
     "karma-osx-reporter": "0.0.4",
     "karma-firework-reporter": "~0.2.3",
-    "karma-coffee-preprocessor": "0.1.3"
+    "karma-coffee-preprocessor": "0.1.3",
+    "compute-cluster": "0.0.7"
   },
   "repository": {
     "type": "git",

script/compress.js

@@ -1,16 +1,28 @@
-var glob = require('glob');
-var fs = require('fs');
-var UglifyJS = require("uglify-js");
+const glob = require('glob');
+const computecluster = require('compute-cluster');
+const path = require('path');
 
-glob.sync("public/optimized/compiled/{*.js,**/*.js}", function (err, files) {
-  console.log('Running UglifyJS on ' + files.length + ' files');
-  files.forEach(function(file) {
-    var result = UglifyJS.minify(file, {
-      screw_ie8: true,
-      source_map: false
-    });
-    fs.writeFileSync(file,result.code, {flag: 'w'}, function(err) {
-      if (err) throw err;
-    });
+var clusterOpts = {
+  module:  path.join(__dirname, "compress_worker.js"),
+  max_backlog: 10000
+}
+// by default, we'll create a cluster as big as the number of cores on your machine,
+// set the CANVAS_BUILD_CONCURRENCY environment variable if you want it to be something else
+if (process.env.CANVAS_BUILD_CONCURRENCY)
+  clusterOpts.max_processes = parseInt(process.env.CANVAS_BUILD_CONCURRENCY);
+
+// allocate a compute cluster
+var cc = new computecluster(clusterOpts);
+
+var files = glob.sync("public/optimized/compiled/{*.js,**/*.js}");
+var toRun = files.length;
+
+for (var i = 0; i < files.length; i++) {
+  cc.enqueue(files[i], function(err, r) {
+    if (err) return console.log("an error occured:", err);
+    if (--toRun === 0) {
+      console.log(files.length + 'files minified')
+      cc.exit();
+    }
   });
-})
+}

script/compress_worker.js

@@ -0,0 +1,14 @@
+const fs = require('fs');
+const UglifyJS = require("uglify-js");
+
+process.on('message', function(filename) {
+  // console.log('minifying ', filename)
+  // var start = new Date();
+  var result = UglifyJS.minify(filename, {
+    screw_ie8: true,
+    source_map: false
+  });
+  fs.writeFileSync(filename, result.code, {flag: 'w'})
+  // console.log('minified   ' + filename + ' in ' + (new Date() - start)/1000 + 's')
+  process.send('complete');
+});