allow migrations to work across job servers

migrations were previously limited to being on the same
job server, now the exported data is uploaded back to
canvas and downloaded again for the importing step

closes #3519

Change-Id: I24deaf4bc1811c4b66b1a2cf79f311ffc1fa9906
Reviewed-on: https://gerrit.instructure.com/2178
Tested-by: Hudson <hudson@instructure.com>
Reviewed-by: Bracken Mosbacker <bracken@instructure.com>

app/models/content_migration.rb

@@ -24,6 +24,7 @@ class ContentMigration < ActiveRecord::Base
   belongs_to :user
   belongs_to :attachment
   belongs_to :overview_attachment, :class_name => 'Attachment'
+  belongs_to :exported_attachment, :class_name => 'Attachment'
   has_a_broadcast_policy
   serialize :migration_settings
   before_save :infer_defaults
@@ -178,20 +179,36 @@ class ContentMigration < ActiveRecord::Base
   def import_content
     self.workflow_state = :importing
     self.save
-    file = File.open(File.join(migration_settings[:export_folder_path], 'course_export.json'))
-    data = JSON.parse(file.read)
-    data = data.with_indifferent_access if data.is_a? Hash
-    migration_settings[:migration_ids_to_import] ||= {:copy=>{}}
-    self.context.import_from_migration(data, migration_settings[:migration_ids_to_import], self)
-  rescue => e
-    self.workflow_state = :failed
-    message = "#{e.to_s}: #{e.backtrace.join("\n")}"
-    migration_settings[:last_error] = message
-    logger.error message
-    self.save
-    raise e
-  ensure
-    clear_migration_data
+    
+    begin
+      @exported_data_zip = download_exported_data
+      @zip_file = Zip::ZipFile.open(@exported_data_zip.path)
+      data = JSON.parse(@zip_file.read('course_export.json'))
+      data = data.with_indifferent_access if data.is_a? Hash
+      
+      if @zip_file.find_entry('all_files.zip')
+        # the file importer needs an actual file to process
+        all_files_path = create_all_files_path(@exported_data_zip.path)
+        @zip_file.extract('all_files.zip', all_files_path)
+        data['all_files_export']['file_path'] = all_files_path
+      else
+        data['all_files_export']['file_path'] = nil if data['all_files_export']
+      end
+      
+      @zip_file.close
+      
+      migration_settings[:migration_ids_to_import] ||= {:copy=>{}}
+      self.context.import_from_migration(data, migration_settings[:migration_ids_to_import], self)
+    rescue => e
+      self.workflow_state = :failed
+      message = "#{e.to_s}: #{e.backtrace.join("\n")}"
+      migration_settings[:last_error] = message
+      logger.error message
+      self.save
+      raise e
+    ensure
+      clear_migration_data
+    end
   end
   handle_asynchronously :import_content, :priority => Delayed::LOW_PRIORITY
   
@@ -199,17 +216,45 @@ class ContentMigration < ActiveRecord::Base
     {:conditions => {:context_id => context.id, :context_type => context.class.to_s} }
   }
   
-  def clear_migration_data
+  def download_exported_data
+    raise "No exported data to import" unless self.exported_attachment
     config = Setting.from_config('external_migration')
-    if !config || !config[:keep_after_complete]
-      if File.exists?(migration_settings[:export_folder_path])
-        begin
-          FileUtils::rm_rf(migration_settings[:export_folder_path])
-        rescue
-          Rails.logger.warn "Couldn't delete export folder for content_migration #{self.id}"
-        end
+    if config && config[:data_folder]
+      @exported_data_zip = Tempfile.new("migration_#{self.id}_", config[:data_folder])
+    else
+      @exported_data_zip = Tempfile.new("migration_#{self.id}_")
+    end
+    
+    if Attachment.local_storage?
+      @exported_data_zip.write File.read(self.exported_attachment.full_filename)
+    elsif Attachment.s3_storage?
+      att = self.exported_attachment
+      require 'aws/s3'
+      AWS::S3::S3Object.stream(att.full_filename, att.bucket_name) do |chunk|
+        @exported_data_zip.write chunk
       end
     end
+    
+    @exported_data_zip.close
+    @exported_data_zip
+  end
+  
+  def create_all_files_path(temp_path)
+    "#{temp_path}_all_files.zip"
+  end
+  
+  def clear_migration_data
+    begin
+      @zip_file.close if @zip_file
+      @zip_file = nil
+      if @exported_data_zip
+        all_files_path = create_all_files_path(@exported_data_zip.path)
+        FileUtils::rm_rf(all_files_path) if File.exists?(all_files_path)
+        @exported_data_zip.unlink
+      end
+    rescue
+      Rails.logger.warn "Couldn't delete files for content_migration #{self.id}"
+    end
   end
 
   def fast_update_progress(val)

config/external_migration.yml.example

@@ -1,14 +1,14 @@
 # This is where temporary files for external migrations will be stored while being processed
 # Possible keys (default value)
 # data_folder (system temp dir)- the path to use (must be writable by daemon processes)
-# keep_after_complete (false) - whether to delete data from system when done 
+# keep_after_complete (false) - whether to delete data from system when done (keeping it around can make debugging easier) 
 
 production:
   keep_after_complete: false
   
 development:
   data_folder: exports/
-  keep_after_complete: true #useful for debugging
+  keep_after_complete: false 
      
 test:
   keep_after_complete: false

db/migrate/20110203205300_add_attachment_to_content_migration.rb

@@ -0,0 +1,9 @@
+class AddAttachmentToContentMigration < ActiveRecord::Migration
+  def self.up
+    add_column :content_migrations, :exported_attachment_id, :integer, :limit => 8
+  end
+
+  def self.down
+    remove_column :content_migrations, :exported_attachment_id
+  end
+end

lib/canvas_migration/migration_worker.rb

@@ -36,4 +36,45 @@ module Canvas::MigrationWorker
     content_migration.save
     att
   end
+
+  def self.upload_exported_data(folder, content_migration)
+    file_name = "exported_data_cm_#{content_migration.id}.zip"
+    zip_file = File.join(folder, file_name)
+    att = nil
+    
+    begin
+      Zip::ZipFile.open(zip_file, 'w') do |zipfile|
+        Dir["#{folder}/**/**"].each do |file|
+          next if File.basename(file) == file_name
+          file_path = file.sub(folder+'/', '')
+          zipfile.add(file_path, file)
+        end
+      end
+
+      upload_file = ActionController::TestUploadedFile.new(zip_file, "application/zip")
+      att = Attachment.new
+      att.context = content_migration
+      att.uploaded_data = upload_file
+      att.save
+      upload_file.unlink
+      content_migration.exported_attachment = att
+      content_migration.save
+    rescue => e
+      content_migration.migration_settings[:last_error] = "#{e.to_s}: #{e.backtrace.join("\n")}"
+      Rails.logger.warn "Error while uploading exported data for content_migration #{content_migration.id} - #{e.to_s}"
+    end
+
+    att
+  end
+  
+  def self.clear_exported_data(folder)
+    begin
+      config = Setting.from_config('external_migration')
+      if !config || !config[:keep_after_complete]
+        FileUtils::rm_rf(folder) if File.exists?(folder)
+      end
+    rescue
+      Rails.logger.warn "Couldn't clear export data for content_migration #{content_migration.id}"
+    end
+  end
 end

lib/canvas_migration/migrator.rb

@@ -44,27 +44,6 @@ class Canvas::Migrator
     raise "Migrator.export should have been overwritten by a sub-class"
   end
 
-  def find_export_dir
-    if @settings[:content_migration_id] && @settings[:user_id]
-      slug = "cm_#{@settings[:content_migration_id]}_user_id_#{@settings[:user_id]}"
-    else
-      slug = "export_#{rand(10000)}"
-    end
-
-    path = create_export_dir(slug)
-    i = 1
-    while File.exists?(path) && File.directory?(path)
-      i += 1
-      path = create_export_dir("#{slug}_attempt_#{i}")
-    end
-
-    path
-  end
-
-  def create_export_dir(slug)
-    File.join(BASE_DOWNLOAD_PATH, @settings[:migration_type], @settings[:course_name].to_s, slug)
-  end
-
   def unzip_archive
     begin
       command = Canvas::MigratorHelper.unzip_command(@archive_file_path, @unzipped_file_path)
@@ -113,7 +92,12 @@ class Canvas::Migrator
   protected
   
   def download_archive
-    temp_file = Tempfile.new("migration")
+    config = Setting.from_config('external_migration')
+    if config && config[:data_folder]
+      temp_file = Tempfile.new("migration", config[:data_folder])
+    else
+      temp_file = Tempfile.new("migration")
+    end
     if @settings[:export_archive_path]
       temp_file.write File.read(@settings[:export_archive_path])
     elsif @settings[:course_archive_download_url] and @settings[:course_archive_download_url] != ""

lib/canvas_migration/migrator_helper.rb

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License along
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-
+require 'tmpdir'
 module Canvas::MigratorHelper
   include Canvas::Migration
 
@@ -25,16 +25,6 @@ module Canvas::MigratorHelper
   
   attr_reader :overview
 
-  # The base directory where all the course data will be download to
-  # The final path for a course will be:
-  # BASE_DOWNLOAD_PATH + blackboard user name + course name
-  if ENV['RAILS_ENV'] and ENV['RAILS_ENV'] == "production"
-    #production path
-    BASE_DOWNLOAD_PATH = "/var/web/migration_tool/data/"
-  else
-    BASE_DOWNLOAD_PATH = "exports/"
-  end
-
   def self.unzip_command(zip_file, dest_dir)
     "unzip -qo #{zip_file.gsub(/ /, "\\ ")} -d #{dest_dir.gsub(/ /, "\\ ")} 2>&1"
   end
@@ -55,6 +45,33 @@ module Canvas::MigratorHelper
 
     error
   end
+  
+  def find_export_dir
+    if @settings[:content_migration_id] && @settings[:user_id]
+      slug = "cm_#{@settings[:content_migration_id]}_user_id_#{@settings[:user_id]}_#{@settings[:migration_type]}"
+    else
+      slug = "export_#{rand(10000)}"
+    end
+
+    path = create_export_dir(slug)
+    i = 1
+    while File.exists?(path) && File.directory?(path)
+      i += 1
+      path = create_export_dir("#{slug}_attempt_#{i}")
+    end
+
+    path
+  end
+
+  def create_export_dir(slug)
+    config = Setting.from_config('external_migration')
+    if config && config[:data_folder]
+      folder = config[:data_folder]
+    else
+      folder = Dir.tmpdir
+    end
+    File.join(folder, slug)
+  end
 
   def make_export_dir
     FileUtils::mkdir_p @base_export_dir

vendor/plugins/qti_exporter/lib/workers/qti_worker.rb

@@ -23,8 +23,11 @@ module Canvas
           file = File.new(overview_file_path)
           Canvas::MigrationWorker::upload_overview_file(file, cm)
         end
+        if export_folder_path
+          Canvas::MigrationWorker::upload_exported_data(export_folder_path, cm)
+          Canvas::MigrationWorker::clear_exported_data(export_folder_path)
+        end
 
-        cm.migration_settings[:export_folder_path] = export_folder_path
         cm.migration_settings[:migration_ids_to_import] = {:copy=>{:assessment_questions=>true}}
         cm.save
         cm.import_content