serve brotli encoded static assets when possible

this should speed things up by reducing the amount of traffic over the
Wire for our JS/CSS/images from our cdn for most browsers/users
(everyone besides IE 11 supports brotli). Should especially help people
on mobile connections and in remote areas.

For example, our vendor webpack bundle went from 850KB to ~500KB

closes: CORE-2755

Test plan:
* with the dev CDN set up in canvs_cdn.yml
* run: RAILS_ENV=production  bundle exec rake canvas:compile_assets
* then run: bundle exec rake canvas:cdn:upload_to_s3
* then run:
  RAILS_ENV=production bin/rake brand_configs:generate_and_upload_all
* then run RAILS_ENV=production bundle exec rails s

now go to canvas in your browser
* from any browser that supports brotli compression, the assets you get
  From the CDN should come from /br/dist/whatever
  (instead of /dist/whatever)
* everything should work the same but you should notice smaller file
  Sizes in the network panel for your js and css assets

Now go to canvas in a browser that doesn’t support brotli, like IE 11
* you should see that it gets its css and js files from
  <cdn host>.com/dist/whatever (and not from /br/dist/whatever)
* you should notice that the assets you are looking at are gzipped
  Just like before, and you can compare against those in chrome and see
  That the gzip version of those files is bigger than the brotli version

Change-Id: I81d28fa31c307d745ecd9b84f1fd55c07fba88ca
Reviewed-on: https://gerrit.instructure.com/188866
Tested-by: Jenkins
Reviewed-by: Cody Cutrer <cody@instructure.com>
QA-Review: Jeremy Putnam <jeremyp@instructure.com>
Product-Review: Ryan Shaw <ryan@instructure.com>

Gemfile.d/app.rb

@@ -57,6 +57,7 @@ gem 'barby', '0.6.5', require: false
   gem 'rqrcode', '0.10.1', require: false
   gem 'chunky_png', '1.3.10', require: false
 gem 'bcrypt', '3.1.11'
+gem 'brotli', '0.2.0', require: false
 gem 'canvas_connect', '0.3.11'
   gem 'adobe_connect', '1.0.8', require: false
 gem 'canvas_webex', '0.17'

app/controllers/application_controller.rb

@@ -136,7 +136,7 @@ class ApplicationController < ActionController::Base
       ]
 
       @js_env = {
-        ASSET_HOST: Canvas::Cdn.config.host,
+        ASSET_HOST: Canvas::Cdn.add_brotli_to_host_if_supported(request),
         active_brand_config_json_url: active_brand_config_url('json'),
         url_to_what_gets_loaded_inside_the_tinymce_editor_css: editor_css,
         url_for_high_contrast_tinymce_editor_css: editor_hc_css,

app/helpers/application_helper.rb

@@ -640,7 +640,7 @@ module ApplicationHelper
   def active_brand_config_url(type, opts={})
     path = active_brand_config(opts).try("public_#{type}_path")
     path ||= BrandableCSS.public_default_path(type, @current_user&.prefers_high_contrast? || opts[:force_high_contrast])
-    "#{Canvas::Cdn.config.host}/#{path}"
+    "#{Canvas::Cdn.add_brotli_to_host_if_supported(request)}/#{path}"
   end
 
   def brand_config_account(opts={})

lib/canvas/cdn.rb

@@ -33,12 +33,22 @@ module Canvas
         source.start_with?('/dist/brandable_css') || Canvas::Cdn::RevManifest.include?(source)
       end
 
-      def asset_host_for(source)
+      def asset_host_for(source, request=nil)
         return unless config.host # unless you've set a :host in the canvas_cdn.yml file, just serve normally
-        config.host if should_be_in_bucket?(source)
+        add_brotli_to_host_if_supported(request) if should_be_in_bucket?(source)
         # Otherwise, return nil & use the same domain the page request came from, like normal.
       end
 
+      def add_brotli_to_host_if_supported(request)
+        # there is a /br/ folder on the s3 bucket that has evertying we publish,
+        # but encoded as brotli instead of gzip
+        "#{config.host}#{'/br' if config.host && supports_brotli?(request)}"
+      end
+
+      def supports_brotli?(request)
+        request && request.headers['Accept-Encoding'].include?('br')
+      end
+
       def push_to_s3!(*args, &block)
         return unless config.bucket
         uploader = Canvas::Cdn::S3Uploader.new(*args)

lib/canvas/cdn/s3_uploader.rb

@@ -16,6 +16,7 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 
 require 'parallel'
+require 'brotli'
 
 module Canvas
   module Cdn
@@ -70,32 +71,42 @@ module Canvas
       def upload_file(remote_path)
         local_path = Pathname.new("#{Rails.public_path}/#{remote_path}")
         return if (local_path.extname == '.gz') || local_path.directory?
-        s3_object = mutex.synchronize { bucket.object(remote_path) }
-        return log("skipping already existing #{remote_path}") if s3_object.exists?
         options = options_for(local_path)
-        s3_object.put(options.merge(body: handle_compression(local_path, options)))
+        {'br' => 'br/', 'gzip' => ''}.each do |compression_type, path_prefix|
+          remote_path_with_prefix = path_prefix + remote_path
+          s3_object = mutex.synchronize { bucket.object(remote_path_with_prefix) }
+          if s3_object.exists?
+            log("skipping already existing #{compression_type} file #{remote_path_with_prefix}")
+          else
+            s3_object.put(options.merge(body: handle_compression(local_path, options, compression_type)))
+          end
+        end
       end
 
       def log(msg)
         Rails.logger.debug "#{self.class} - #{msg}"
       end
 
-      def handle_compression(file, options)
-        if file.size > 150 # gzipping small files is not worth it
-          gzipped = ActiveSupport::Gzip.compress(file.read, Zlib::BEST_COMPRESSION)
-          compression = 100 - (100.0 * gzipped.size / file.size).round
-          # if we couldn't compress more than 5%, the gzip decoding cost to the
-          # client makes it is not worth serving gzipped
+      def handle_compression(file, options, compression_algorithm='gzip')
+        contents = file.binread
+        if file.size > 150 # compressing small files is not worth it
+          compressed = if compression_algorithm == 'br'
+            Brotli.deflate(contents, quality: 11)
+          elsif compression_algorithm == 'gzip'
+            ActiveSupport::Gzip.compress(contents, Zlib::BEST_COMPRESSION)
+          end
+          compression = 100 - (100.0 * compressed.size / file.size).round
+          # if we couldn't compress more than 5%, the gzip/brotli decoding cost to the
+          # client makes it not worth serving compressed
           if compression > 5
-            options[:content_encoding] = 'gzip'
-            log "uploading gzipped #{file}. was: #{file.size} now: #{gzipped.size} saved: #{compression}%"
-            return gzipped
+            options[:content_encoding] = compression_algorithm
+            log "uploading #{compression_algorithm}'ed #{file}. was: #{file.size} now: #{compressed.size} saved: #{compression}%"
+            return compressed
           end
         end
-        log "uploading ungzipped #{file}"
-        file.read
+        log "uploading un-#{compression_algorithm}'ed #{file}"
+        contents
       end
-
     end
   end
 end

spec/lib/canvas/cdn_spec.rb

@@ -19,15 +19,16 @@
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper.rb')
 
 describe Canvas::Cdn do
+
+  before :each do
+    @original_config = Canvas::Cdn.config.dup
+  end
+
+  after :each do
+    Canvas::Cdn.config.replace(@original_config)
+  end
+
   describe '.enabled?' do
-    before :each do
-      @original_config = Canvas::Cdn.config.dup
-    end
-
-    after :each do
-      Canvas::Cdn.config.replace(@original_config)
-    end
-
     it 'returns true when the cdn config has a bucket' do
       Canvas::Cdn.config.merge! enabled: true, bucket: 'bucket_name'
       expect(Canvas::Cdn.enabled?).to eq true
@@ -38,4 +39,38 @@ describe Canvas::Cdn do
       expect(Canvas::Cdn.enabled?).to eq false
     end
   end
+
+  describe '.add_brotli_to_host_if_supported' do
+    it 'puts a /br on the front when brotli is supported' do
+      Canvas::Cdn.config.merge! host: 'somehostname'
+      request = double()
+      expect(request).to receive(:headers).and_return({'Accept-Encoding'=> 'gzip, deflate, br'})
+      expect(Canvas::Cdn.add_brotli_to_host_if_supported(request)).to eq "somehostname/br"
+    end
+
+    it 'does not put a /br on the front when brotli is not supported' do
+      Canvas::Cdn.config.merge! host: 'somehostname'
+      request = double()
+      expect(request).to receive(:headers).and_return({'Accept-Encoding'=> 'gzip, deflate'})
+      expect(Canvas::Cdn.add_brotli_to_host_if_supported(request)).to eq "somehostname"
+    end
+  end
+
+  describe '.supports_brotli?' do
+    it 'returns false when there is no request avaiable' do
+      expect(Canvas::Cdn.supports_brotli?(nil)).to be_falsy
+    end
+
+    it 'returns false if user agent doesnt accept-encoding "br"' do
+      request = double()
+      expect(request).to receive(:headers).and_return({'Accept-Encoding' => 'gzip, deflate'})
+      expect(Canvas::Cdn.supports_brotli?(request)).to be_falsy
+    end
+
+    it 'returns true when the user agent supports brotli' do
+      request = double()
+      expect(request).to receive(:headers).and_return({'Accept-Encoding'=> 'gzip, deflate, br'})
+      expect(Canvas::Cdn.supports_brotli?(request)).to be_truthy
+    end
+  end
 end