diff --git a/app/models/attachment.rb b/app/models/attachment.rb index 7faf3973452..3e6949664da 100644 --- a/app/models/attachment.rb +++ b/app/models/attachment.rb @@ -512,6 +512,14 @@ class Attachment < ActiveRecord::Base end end + def root_account + begin + root_account_id && Account.find_cached(root_account_id) + rescue ::Canvas::AccountCacheError + nil + end + end + def namespace read_attribute(:namespace) || (new_record? ? write_attribute(:namespace, infer_namespace) : nil) end @@ -836,11 +844,7 @@ class Attachment < ActiveRecord::Base end def url_ttl - settings = begin - root_account_id && Account.find_cached(root_account_id).settings - rescue ::Canvas::AccountCacheError - end - setting = settings&.[](:s3_url_ttl_seconds) + setting = root_account&.settings&.[](:s3_url_ttl_seconds) setting ||= Setting.get('attachment_url_ttl', 1.hour.to_s) setting.to_i.seconds end diff --git a/app/views/plugins/_inst_fs_settings.html.erb b/app/views/plugins/_inst_fs_settings.html.erb index 056d495cdb9..7fa217063ee 100644 --- a/app/views/plugins/_inst_fs_settings.html.erb +++ b/app/views/plugins/_inst_fs_settings.html.erb @@ -17,4 +17,20 @@ %> <%= fields_for :settings, OpenObject.new(settings) do |f| %> + + + + + + + + +
+

<%= t(:description, <<-TEXT) + Hosts uploaded files with the Inst-FS service. + TEXT + %>

+
<%= f.blabel :migration_rate, :en => "Chance of migrating older files into Inst-FS on access (0 to 100)" %> + <%= f.text_field :migration_rate %> +
<% end %> diff --git a/lib/canvas/plugins/default_plugins.rb b/lib/canvas/plugins/default_plugins.rb index 7968a0b282b..5f9c286cce6 100644 --- a/lib/canvas/plugins/default_plugins.rb +++ b/lib/canvas/plugins/default_plugins.rb @@ -405,8 +405,11 @@ Canvas::Plugin.register('inst_fs', nil, { :author => 'Instructure', :author_website => 'http://www.instructure.com', :version => '0.0.1', - :settings => nil, - :settings_partial => 'plugins/inst_fs_settings' + :settings => { + :migration_rate => 0, + }, + :settings_partial => 'plugins/inst_fs_settings', + :validator => 'InstFsValidator' }) Canvas::Plugin.register('unsplash', nil, { name: lambda{ t :name, 'Unsplash' }, diff --git a/lib/canvas/plugins/validators/inst_fs_validator.rb b/lib/canvas/plugins/validators/inst_fs_validator.rb new file mode 100644 index 00000000000..ecfb4054727 --- /dev/null +++ b/lib/canvas/plugins/validators/inst_fs_validator.rb @@ -0,0 +1,33 @@ +# +# Copyright (C) 2020 - present Instructure, Inc. +# +# This file is part of Canvas. +# +# Canvas is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation, version 3 of the License. +# +# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see . +# + +module Canvas::Plugins::Validators::InstFsValidator + def self.validate(settings, plugin_setting) + if settings[:migration_rate].blank? + migration_rate = 0 + else + migration_rate = settings[:migration_rate].to_f rescue nil + end + if migration_rate.nil? || migration_rate < 0 || migration_rate > 100 + plugin_setting.errors.add(:base, I18n.t('Please enter a number between 0 and 100 for the migration rate')) + return false + end + settings[:migration] = migration_rate + settings.slice(:migration_rate).to_h.with_indifferent_access + end +end diff --git a/lib/file_authenticator.rb b/lib/file_authenticator.rb index afc8351f66f..305b53bfbdd 100644 --- a/lib/file_authenticator.rb +++ b/lib/file_authenticator.rb @@ -51,19 +51,9 @@ class FileAuthenticator Digest::MD5.hexdigest("#{@user&.global_id}|#{@acting_as&.global_id}|#{@oauth_host}") end - def instfs_options(attachment, extras={}) - { - user: @user, - acting_as: @acting_as, - access_token: @access_token, - root_account: @root_account, - oauth_host: @oauth_host, - expires_in: attachment.url_ttl, - }.merge(extras) - end - def download_url(attachment) return nil unless attachment + migrate_legacy_attachment_to_instfs(attachment) if attachment.instfs_hosted? options = instfs_options(attachment, download: true) InstFS.authenticated_url(attachment, options) @@ -75,6 +65,7 @@ class FileAuthenticator def inline_url(attachment) return nil unless attachment + migrate_legacy_attachment_to_instfs(attachment) if attachment.instfs_hosted? options = instfs_options(attachment, download: false) InstFS.authenticated_url(attachment, options) @@ -93,4 +84,32 @@ class FileAuthenticator attachment.thumbnail_url(options) end end + + private + + def migrate_legacy_attachment_to_instfs(attachment) + # on-demand migration to inst-fs (if necessary and opted into). this is a + # server to server communication during the request (just before redirect, + # typically), but it's only a POST of the metadata to inst-fs, so should be + # quick, which we enforce with a timeout; inst-fs will "naturalize" the + # object contents later out-of-band + return unless InstFS.migrate_attachment?(attachment) + attachment.instfs_uuid = InstFS.export_reference(attachment) + attachment.save! + rescue InstFS::ExportReferenceError + # in the case of a recognized error exporting reference, just ignore, + # acting as if we didn't intend to migrate in the first place, rather than + # interrupting what could be a successful redirect to s3 + end + + def instfs_options(attachment, extras={}) + { + user: @user, + acting_as: @acting_as, + access_token: @access_token, + root_account: @root_account, + oauth_host: @oauth_host, + expires_in: attachment.url_ttl, + }.merge(extras) + end end diff --git a/lib/inst_fs.rb b/lib/inst_fs.rb index e372f46b2b5..115580be947 100644 --- a/lib/inst_fs.rb +++ b/lib/inst_fs.rb @@ -22,6 +22,14 @@ module InstFS Canvas::Plugin.find('inst_fs').enabled? && !!app_host && !!jwt_secret end + def check_migration_rate? + rand < Canvas::Plugin.find('inst_fs').settings[:migration_rate].to_f / 100.0 + end + + def migrate_attachment?(attachment) + enabled? && !attachment.instfs_hosted? && Attachment.s3_storage? && check_migration_rate? + end + def login_pixel(user, session, oauth_host) return if session[:oauth2] # don't stomp an existing oauth flow in progress return if session[:pending_otp] @@ -153,6 +161,60 @@ module InstFS raise InstFS::DirectUploadError, "received code \"#{response.code}\" from service, with message \"#{response.body}\"" end + def export_reference(attachment) + raise InstFS::ExportReferenceError, "attachment already has instfs_uuid" if attachment.instfs_hosted? + raise InstFS::ExportReferenceError, "can't export non-s3 attachments to inst-fs" unless Attachment.s3_storage? + + # compare to s3_bucket_url in the aws-sdk-s3 gem's + # lib/aws-sdk-s3/customizations/bucket.rb; we're leaving out the bucket + # name from the url. otherwise, this is effectively + # `attachment.bucket.url` + s3_client = attachment.bucket.client + s3_url = s3_client.config.endpoint.dup + if s3_client.config.region == 'us-east-1' && + s3_client.config.s3_us_east_1_regional_endpoint == 'legacy' + s3_url.host = Aws::S3::Plugins::IADRegionalEndpoint.legacy_host(s3_url.host) + end + + body = { + objectStore: { + type: "s3", + params: { + host: s3_url.to_s, + bucket: attachment.bucket.name + } + }, + # single reference + references: [{ + storeKey: attachment.full_filename, + timestamp: attachment.created_at.to_i, + filename: attachment.filename, + displayName: attachment.display_name, + content_type: attachment.content_type, + encoding: attachment.encoding, + size: attachment.size, + user_id: attachment.context_user&.global_id&.to_s, + root_account_id: attachment.root_account&.global_id&.to_s, + sha512: nil, # to be calculated by inst-fs + }] + }.to_json + + response = CanvasHttp.post(export_references_url, body: body, content_type: "application/json") + raise InstFS::ExportReferenceError, "received code \"#{response.code}\" from service, with message \"#{response.body}\"" unless response.class == Net::HTTPOK + + json_response = JSON.parse(response.body) + well_formed = + json_response.is_a?(Hash) && + json_response.key?("success") && + json_response["success"].is_a?(Array) && + json_response["success"].length == 1 && + json_response["success"][0].is_a?(Hash) + json_response["success"][0].key?("id") + raise InstFS::ExportReferenceError, "import succeeded, but response body did not have expected shape" unless well_formed + + json_response["success"][0]["id"] + end + def duplicate_file(instfs_uuid) token = duplicate_file_jwt(instfs_uuid) url = "#{app_host}/files/#{instfs_uuid}/duplicate?token=#{token}" @@ -387,6 +449,7 @@ module InstFS end class DirectUploadError < StandardError; end + class ExportReferenceError < StandardError; end class DuplicationError < StandardError; end class DeletionError < StandardError; end end