ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

direct-to-s3 server side changes 

Change-Id: Ie7b415b84f403c98d82f0e67212ae2e7b051b67d
Reviewed-on: https://gerrit.instructure.com/2096
Tested-by: Hudson <hudson@instructure.com>
Reviewed-by: Brian Whitmer <brian@instructure.com>
This commit is contained in:
JT Olds 2011-02-01 11:50:10 -07:00
parent 9168dfbdec
commit ba735d41b6
8 changed files with 286 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
app/controllers/files_controller.rb
View File

@ -16,8 +16,11 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
CONTENT_LENGTH_RANGE = 50*1024*1024
S3_EXPIRATION_TIME = 30.minutes
class FilesController < ApplicationController
before_filter :require_context, :except => [:public_feed,:full_index,:assessment_question_show, :image_thumbnail]
before_filter :require_context, :except => [:public_feed,:full_index,:assessment_question_show, :image_thumbnail,:create_pending,:s3_success]
before_filter :check_file_access_flags, :only => :show_relative
before_filter { |c| c.active_tab = "files" }
@ -353,6 +356,163 @@ class FilesController < ApplicationController
if authorized_action(@attachment, @current_user, :create)
end
end
def create_pending
@context = Context.find_by_asset_string(params[:attachment][:context_code])
@attachment = Attachment.new
@attachment.context = @context
permission_object = @attachment
permission = :create
# Using workflow_state we can keep track of the files that have been built
# but we don't know that there's an s3 component for yet (it's still being
# uploaded)
workflow_state = 'unattached'
# There are multiple reasons why we could be building a file. The default
# is to upload it to a context. In the other cases we need to check the
# permission related to the purpose to make sure the file isn't being
# uploaded just to disappear later
if @context.is_a?(Assignment) && params[:intent] == 'comment'
permission_object = @context
permission = :attach_submission_comment_files
elsif @context.is_a?(Assignment) && params[:intent] == 'submit'
permission_object = @context
permission = (@context.submission_types || "").match(/online_file_upload/) ? :submit : :nothing
elsif @context.respond_to?(:is_a_context) && params[:intent] == 'attach_discussion_file'
permission_object = @context.discussion_topics.new
permission = :attach
elsif @context.respond_to?(:is_a_context) && params[:intent] == 'message'
permission_object = @context
permission = :send_messages
workflow_state = 'unattached_temporary'
elsif @context.respond_to?(:is_a_context) && params[:intent] && params[:intent] != 'upload'
# In other cases (like unzipping a file, extracting a QTI, etc.
# we don't actually want the uploaded file to show up in the context's
# file listings. If you set its workflow_state to unattached_temporary
# then it will never be activated.
workflow_state = 'unattached_temporary'
end
if authorized_action(permission_object, @current_user, permission)
if @context.respond_to?(:is_a_context) && params[:intent] == 'upload'
get_quota
return if quota_exceeded(named_context_url(@context, :context_files_url))
end
@attachment.filename = params[:attachment][:filename]
@attachment.file_state = 'deleted'
@attachment.workflow_state = workflow_state
if @context.respond_to?(:folders)
@folder = @context.folders.active.find_by_id(params[:attachment][:folder_id])
@folder ||= Folder.unfiled_folder(@context)
@attachment.folder_id = @folder.id
end
@attachment.content_type = Attachment.mimetype(@attachment.filename)
@attachment.save!
if Attachment.s3_storage?
# Build the data that will be needed for the user to upload to s3
# without us being the middle-man
full_filename = @attachment.full_filename.gsub(/\+/, " ")
policy = {
'expiration' => S3_EXPIRATION_TIME.from_now.utc.iso8601,
'conditions' => [
{'bucket' => @attachment.bucket_name},
{'key' => full_filename},
{'acl' => 'private'},
['starts-with', '$Filename', ''],
['starts-with', '$folder', ''],
['content-length-range', 1, CONTENT_LENGTH_RANGE]
]
}
if params[:s3_no_redirect]
policy['conditions'] << {'success_action_status' => '201'}
else
policy['conditions'] << {'success_action_redirect' => s3_success_url(@attachment.id, :uuid => @attachment.uuid)}
end
if @attachment.content_type && @attachment.content_type != "unknown/unknown"
policy['conditions'] << {'content-type' => @attachment.content_type}
end
policy_encoded = Base64.encode64(policy.to_json).gsub(/\n/, '')
signature = Base64.encode64(
OpenSSL::HMAC.digest(
OpenSSL::Digest::Digest.new('sha1'), AWS::S3::Base.connection.secret_access_key, policy_encoded
)
).gsub(/\n/, '')
res = {
:id => @attachment.id,
:upload_url => "http://#{@attachment.bucket_name}.s3.amazonaws.com/",
:proxied_upload_url => nil,
:file_param => 'file',
:remote_url => true,
:success_url => s3_success_url(@attachment.id, :uuid => @attachment.uuid),
:upload_params => {
'AWSAccessKeyId' => AWS::S3::Base.connection.access_key_id,
'key' => full_filename,
'Policy' => policy_encoded,
'Filename' => '',
'folder' => '',
'acl' => 'private',
'Signature' => signature
}
}
if params[:s3_no_redirect]
res[:upload_params]['success_action_status'] = '201'
else
res[:upload_params]['success_action_redirect'] = s3_success_url(@attachment.id, :uuid => @attachment.uuid)
end
if @attachment.content_type && @attachment.content_type != "unknown/unknown"
res[:upload_params]['Content-Type'] = @attachment.content_type
end
render :json => res.to_json
elsif Attachment.local_storage?
render :json => {
:id => @attachment.id,
:upload_url => named_context_url(@context, :context_files_url, :format => :text) + "?" + { ActionController::Base.session_options[:key] => request.session_options[:id] }.to_query,
:remote_url => false,
:file_param => 'attachment[uploaded_data]', #uploadify ignores this and uses 'file'
:upload_params => {
'attachment[folder_id]' => params[:attachment][:folder_id] || '',
'folder' => '',
'Filename' => '',
'attachment[unattached_attachment_id]' => @attachment.id,
'authenticity_token' => form_authenticity_token
}
}.to_json
else
raise "Unknown storage system configured"
end
end
end
def s3_success
@attachment = Attachment.find_by_id_and_workflow_state_and_uuid(params[:id], 'unattached', params[:uuid])
details = AWS::S3::S3Object.about(@attachment.full_filename, @attachment.bucket_name) rescue nil
if @attachment && details
unless @attachment.workflow_state == 'unattached_temporary'
@attachment.workflow_state = nil
@attachment.file_state = 'available'
end
@attachment.md5 = (details['etag'] || "").gsub(/\"/, '')
@attachment.content_type = details['content-type']
@attachment.size = details['content-length']
if @attachment.md5 && !@attachment.md5.empty? && ns = @attachment.infer_namespace
if existing_attachment = Attachment.find_all_by_md5_and_namespace(@attachment.md5, ns).detect{|a| a.id != @attachment.id && !a.root_attachment_id && a.content_type == @attachment.content_type }
AWS::S3::S3Object.delete(@attachment.full_filename, @attachment.bucket_name) rescue nil
@attachment.root_attachment = existing_attachment
@attachment.write_attribute(:filename, nil)
@attachment.clear_cached_urls
end
end
@attachment.save!
@attachment.submit_to_scribd!
render_for_text @attachment.to_json(:allow => :uuid, :methods => [:uuid,:readable_size,:mime_class,:currently_locked,:scribdable?], :permissions => {:user => @current_user, :session => session})
else
render_for_text ""
end
end
# POST /files
# POST /files.xml
@ -360,15 +520,19 @@ class FilesController < ApplicationController
@folder = @context.folders.active.find_by_id(params[:attachment].delete(:folder_id))
@folder ||= Folder.unfiled_folder(@context)
params[:attachment][:uploaded_data] ||= params[:attachment_uploaded_data]
params[:attachment][:uploaded_data] ||= params[:file]
params[:attachment][:user] = @current_user
params[:attachment].delete :context_id
params[:attachment].delete :context_type
@attachment = @context.attachments.new #(params[:attachment])
@attachment = @context.attachments.find_by_id_and_workflow_state(params[:attachment].delete(:unattached_attachment_id), 'unattached')
@attachment ||= @context.attachments.new #(params[:attachment])
if authorized_action(@attachment, @current_user, :create)
get_quota
return if quota_exceeded(named_context_url(@context, :context_files_url))
respond_to do |format|
@attachment.folder_id = @folder.id
@attachment.folder_id ||= @folder.id
@attachment.workflow_state = nil
@attachment.file_state = 'available'
success = nil
if params[:attachment] && params[:attachment][:source_attachment_id]
a = Attachment.find(params[:attachment].delete(:source_attachment_id))
@ -380,7 +544,12 @@ class FilesController < ApplicationController
success = @attachment.save
end
end
success = @attachment.update_attributes(params[:attachment]) if params[:attachment][:uploaded_data]
if params[:attachment][:uploaded_data]
success = @attachment.update_attributes(params[:attachment])
@attachment.errors.add_to_base("Upload failed, server error, please try again.") unless success
else
@attachment.errors.add_to_base("Upload failed, expected form field missing")
end
unless (@attachment.cacheable_s3_url rescue nil)
success = false
if (params[:attachment][:uploaded_data].size == 0 rescue false)

1
app/controllers/submissions_controller.rb
View File

@ -111,6 +111,7 @@ class SubmissionsController < ApplicationController
attachment_ids.each do |id|
params[:submission][:attachments] << @current_user.attachments.active.find_by_id(id) if @current_user
params[:submission][:attachments] << @group.attachments.active.find_by_id(id) if @group
params[:submission][:attachments].compact!
end
if params[:attachments] && params[:submission][:submission_type] == 'online_upload'

6
app/models/assignment.rb
View File

@ -628,7 +628,7 @@ class Assignment < ActiveRecord::Base
self.cached_context_grants_right?(user, session, :participate_as_student) &&
!self.locked_for?(user)
}
set { can :submit }
set { can :submit and can :attach_submission_comment_files }
given { |user, session| !self.locked_for?(user) &&
(self.context.allow_student_assignment_edits rescue false) &&
@ -637,10 +637,10 @@ class Assignment < ActiveRecord::Base
set { can :update_content }
given { |user, session| self.cached_context_grants_right?(user, session, :manage_grades) }#self.context.admins.find_by_id(user) }
set { can :update and can :update_content and can :grade and can :delete and can :create and can :read }
set { can :update and can :update_content and can :grade and can :delete and can :create and can :read and can :attach_submission_comment_files }
given { |user, session| self.cached_context_grants_right?(user, session, :manage_assignments) }#self.context.admins.find_by_id(user) }
set { can :update and can :update_content and can :delete and can :create and can :read }
set { can :update and can :update_content and can :delete and can :create and can :read and can :attach_submission_comment_files }
end
def self.search(query)

22
app/models/attachment.rb
View File

@ -324,7 +324,7 @@ class Attachment < ActiveRecord::Base
def inline_content?
self.content_type.match(/\Atext/) || self.extension == '.html' || self.extension == '.htm' || self.extension == '.swf'
end
def self.s3_config
# Return existing value, even if nil, as long as it's defined
return @s3_config if defined?(@s3_config)
@ -338,11 +338,13 @@ class Attachment < ActiveRecord::Base
end
def self.s3_storage?
file_store_config['storage'] == 's3' && s3_config
file_store_config['storage'] == 's3' && s3_config || (RAILS_ENV == "test" && Setting.get("file_storage_test_override", nil) == "s3")
end
def self.local_storage?
file_store_config['storage'] == 'local' || !s3_storage?
rv = !s3_storage?
raise "Unknown storage type!" if rv && file_store_config['storage'] != 'local'
rv
end
def downloadable?
@ -368,6 +370,9 @@ class Attachment < ActiveRecord::Base
self.attachment_fu_filename = val
end
end
def bucket_name; "no-bucket"; end
def after_attachment_saved
# No point in submitting to scribd since there's not a reliable
# URL to provide for referencing
@ -475,6 +480,12 @@ class Attachment < ActiveRecord::Base
h.number_to_human_size(self.size) rescue "size unknown"
end
def clear_cached_urls
self.cached_s3_url = nil
self.s3_url_cached_at = nil
self.cached_scribd_thumbnail = nil
end
def cacheable_s3_url
cached = cached_s3_url && s3_url_cached_at && s3_url_cached_at >= (Time.now - 24.hours.to_i)
if !cached
@ -598,6 +609,9 @@ class Attachment < ActiveRecord::Base
given { |user, session| self.cached_context_grants_right?(user, session, :read) && !self.locked_for?(user) } #students.include? user }
set { can :download }
given { |user, session| self.context_type == 'Submission' && self.context.grant_rights?(user, session, :comment) }
set { can :create }
given { |user, session|
u = session && User.find_by_id(session['file_access_user_id'])
u && self.cached_context_grants_right?(u, session, :read) &&
@ -691,6 +705,8 @@ class Attachment < ActiveRecord::Base
state :to_be_zipped
state :zipping
state :zipped
state :unattached
state :unattached_temporary
end
named_scope :to_be_zipped, lambda{

2
app/models/folder.rb
View File

@ -24,7 +24,7 @@ class Folder < ActiveRecord::Base
belongs_to :cloned_item
belongs_to :parent_folder, :class_name => "Folder"
has_many :file_attachments, :class_name => "Attachment", :order => 'position'
has_many :active_file_attachments, :class_name => 'Attachment', :conditions => ['attachments.file_state in (?, ?, ?)', 'available', 'public', 'hidden'], :order => 'position, display_name'
has_many :active_file_attachments, :class_name => 'Attachment', :conditions => ['attachments.file_state != ? AND attachments.file_state != ?', 'deleted', 'hidden'], :order => 'position, display_name'
has_many :visible_file_attachments, :class_name => 'Attachment', :conditions => ['attachments.file_state in (?, ?)', 'available', 'public'], :order => 'position, display_name'
has_many :sub_folders, :class_name => "Folder", :foreign_key => "parent_folder_id", :dependent => :destroy, :order => 'position'
has_many :active_sub_folders, :class_name => "Folder", :conditions => ['folders.workflow_state != ?', 'deleted'], :foreign_key => "parent_folder_id", :dependent => :destroy, :order => 'position'

2
app/models/submission.rb
View File

@ -507,7 +507,7 @@ class Submission < ActiveRecord::Base
# since they're all being held on the assignment for now.
attachments ||= []
old_ids = (Array(self.attachment_ids || "").join(",")).split(",").map{|id| id.to_i}
write_attribute(:attachment_ids, attachments.select{|a| old_ids.include?(a.id) || (a.recently_created? && a.context == self.assignment) || a.context != self.assignment }.map{|a| a.id}.join(","))
write_attribute(:attachment_ids, attachments.select{|a| a && a.id && old_ids.include?(a.id) || (a.recently_created? && a.context == self.assignment) || a.context != self.assignment }.map{|a| a.id}.join(","))
end
def validate_single_submission

2
config/routes.rb
View File

@ -535,6 +535,8 @@ ActionController::Routing::Routes.draw do |map|
map.calendar 'calendar', :controller => 'calendars', :action => 'show', :conditions => { :method => :get }
map.files 'files', :controller => 'files', :action => 'full_index', :conditions => { :method => :get }
map.s3_success 'files/s3_success/:id', :controller => 'files', :action => 's3_success'
map.file_create_pending 'files/pending', :controller=> 'files', :action => 'create_pending'
map.assignments 'assignments', :controller => 'assignments', :action => 'index', :conditions => { :method => :get }
# The priority is based upon order of creation: first created -> highest priority.

86
spec/controllers/files_controller_spec.rb
View File

@ -235,4 +235,90 @@ describe FilesController do
# assigns[:attachment].should be_frozen
end
end
describe "POST 'create_pending'" do
it "should require authorization" do
course_with_teacher(:active_all => true)
post 'create_pending', {:attachment => {:context_code => @course.asset_string}}
assert_unauthorized
end
it "should create file placeholder (in local mode)" do
class Attachment
class <<self
alias :old_s3_storage? :s3_storage?
alias :old_local_storage? :local_storage?
end
def self.s3_storage?; false; end
def self.local_storage?; true; end
end
begin
Attachment.local_storage?.should eql(true)
Attachment.s3_storage?.should eql(false)
course_with_teacher_logged_in(:active_all => true)
post 'create_pending', {:attachment => {
:context_code => @course.asset_string,
:filename => "bob.txt"
}}
response.should be_success
assigns[:attachment].should_not be_nil
assigns[:attachment].id.should_not be_nil
json = JSON.parse(response.body) rescue nil
json.should_not be_nil
json['id'].should eql(assigns[:attachment].id)
json['upload_url'].should_not be_nil
json['upload_params'].should_not be_nil
json['upload_params'].should_not be_empty
json['remote_url'].should eql(false)
ensure
class Attachment
class <<self
alias :s3_storage? :old_s3_storage?
alias :local_storage? :old_local_storage?
end
end
end
end
it "should create file placeholder (in s3 mode)" do
class Attachment
class <<self
alias :old_s3_storage? :s3_storage?
alias :old_local_storage? :local_storage?
end
def self.s3_storage?; true; end
def self.local_storage?; false; end
end
begin
Attachment.s3_storage?.should eql(true)
Attachment.local_storage?.should eql(false)
course_with_teacher_logged_in(:active_all => true)
post 'create_pending', {:attachment => {
:context_code => @course.asset_string,
:filename => "bob.txt"
}}
response.should be_success
assigns[:attachment].should_not be_nil
assigns[:attachment].id.should_not be_nil
json = JSON.parse(response.body) rescue nil
json.should_not be_nil
json['id'].should eql(assigns[:attachment].id)
json['upload_url'].should_not be_nil
json['upload_params'].should_not be_nil
json['upload_params'].should_not be_empty
json['remote_url'].should eql(true)
ensure
class Attachment
class <<self
alias :s3_storage? :old_s3_storage?
alias :local_storage? :old_local_storage?
end
end
end
end
end
describe "POST 's3_success'" do
end
end