expire content migration jobs after 48 hours

test plan:
 - in a console, do
   Setting.set('content_migration_job_expiration_hours', -1)
 - try to do course copies and course imports.
   they should fail with "job expired" errors.
 - go back to the console and undo the expire-migration-jobs-
   -in-the-past setting with the following:
   Setting.remove('content_migration_job_expiration_hours')
 - now migration jobs should work again

fixes CNVS-25113

Change-Id: I2578c3e124276a950b73b4402f30625d8738a6c3
Reviewed-on: https://gerrit.instructure.com/68867
Tested-by: Jenkins
Reviewed-by: James Williams  <jamesw@instructure.com>
Reviewed-by: Simon Williams <simon@instructure.com>
QA-Review: Jahnavi Yetukuri <jyetukuri@instructure.com>
Product-Review: Simon Williams <simon@instructure.com>

Gemfile.d/app.rb

@@ -34,7 +34,7 @@ gem 'bcrypt-ruby', '3.0.1'
 gem 'canvas_connect', '0.3.8'
   gem 'adobe_connect', '1.0.3', require: false
 gem 'canvas_webex', '0.15'
-gem 'canvas-jobs', '0.10.0'
+gem 'canvas-jobs', '0.10.2'
   gem 'rufus-scheduler', '3.1.2', require: false
 gem 'ffi', '1.1.5', require: false
 gem 'hairtrigger', '0.2.15'

app/models/content_migration.rb

@@ -316,7 +316,8 @@ class ContentMigration < ActiveRecord::Base
     set_default_settings
     plugin ||= Canvas::Plugin.find(migration_type)
     if plugin
-      queue_opts = {:priority => Delayed::LOW_PRIORITY, :max_attempts => 1}
+      queue_opts = {:priority => Delayed::LOW_PRIORITY, :max_attempts => 1,
+                    :expires_at => Setting.get('content_migration_job_expiration_hours', '48').to_i.hours.from_now}
       if self.strand
         queue_opts[:strand] = self.strand
       else
@@ -327,7 +328,7 @@ class ContentMigration < ActiveRecord::Base
         # it's ready to be imported
         self.workflow_state = :importing
         self.save
-        self.send_later_enqueue_args(:import_content, queue_opts)
+        self.send_later_enqueue_args(:import_content, queue_opts.merge(:on_permanent_failure => :fail_with_error!))
       else
         # find worker and queue for conversion
         begin

db/migrate/20151123210429_add_expires_at_to_jobs.rb

@@ -0,0 +1,17 @@
+class AddExpiresAtToJobs < ActiveRecord::Migration
+  tag :predeploy
+
+  def connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def up
+    add_column :delayed_jobs, :expires_at, :datetime
+    add_column :failed_jobs, :expires_at, :datetime
+  end
+
+  def down
+    remove_column :delayed_jobs, :expires_at
+    remove_column :failed_jobs, :expires_at
+  end
+end

gems/plugins/qti_exporter/lib/canvas/migration/worker/qti_worker.rb

@@ -1,6 +1,6 @@
 module Canvas::Migration
   module Worker
-    class QtiWorker < Struct.new(:migration_id)
+    class QtiWorker < Base
 
       def perform
         cm = ContentMigration.where(id: migration_id).first

lib/canvas/migration/worker.rb

@@ -20,6 +20,15 @@ require 'action_controller_test_process'
 
 module Canvas::Migration::Worker
 
+  class Base < Struct.new(:migration_id)
+    def on_permanent_failure(error)
+      if migration_id
+        cm = ContentMigration.where(id: migration_id).first
+        cm.fail_with_error!(error) if cm
+      end
+    end
+  end
+
   def self.get_converter(settings)
     Canvas::Migration::Archive.new(settings).get_converter
   end

lib/canvas/migration/worker/course_copy_worker.rb

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License along
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-class Canvas::Migration::Worker::CourseCopyWorker < Struct.new(:migration_id)
+class Canvas::Migration::Worker::CourseCopyWorker < Canvas::Migration::Worker::Base
   def perform(cm=nil)
     cm ||= ContentMigration.find migration_id
 

lib/canvas/migration/worker/zip_file_worker.rb

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License along
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-class Canvas::Migration::Worker::ZipFileWorker < Struct.new(:migration_id)
+class Canvas::Migration::Worker::ZipFileWorker < Canvas::Migration::Worker::Base
   def perform(cm=nil)
     cm ||= ContentMigration.find migration_id
 

lib/cc/importer/cc_worker.rb

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License along
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-class Canvas::Migration::Worker::CCWorker < Struct.new(:migration_id)
+class Canvas::Migration::Worker::CCWorker < Canvas::Migration::Worker::Base
   def perform(cm=nil)
     cm ||= ContentMigration.where(id: migration_id).first
     cm.job_progress.start unless cm.skip_job_progress

spec/models/content_migration_spec.rb

@@ -428,4 +428,43 @@ describe ContentMigration do
     expect(qq.question_data.to_yaml).to include("/media_objects/m-5U5Jww6HL7zG35CgyaYGyA5bhzsremxY")
 
   end
+
+  it "expires migration jobs after 48 hours" do
+    course_with_teacher
+    cm = ContentMigration.new(:context => @course, :user => @teacher)
+    cm.migration_type = 'common_cartridge_importer'
+    cm.workflow_state = 'created'
+    cm.save!
+    cm.queue_migration
+
+    Canvas::Migration::Worker::CCWorker.any_instance.expects(:perform).never
+    Timecop.travel(50.hours.from_now) do
+      run_jobs
+    end
+
+    cm.reload
+    expect(cm).to be_failed
+    expect(cm.migration_issues).not_to be_empty
+    expect(cm.migration_issues.last.error_report.message).to include 'job expired'
+  end
+
+  it "expires import jobs after 48 hours" do
+    course_with_teacher
+    cm = ContentMigration.new(:context => @course, :user => @teacher)
+    cm.migration_type = 'common_cartridge_importer'
+    cm.workflow_state = 'exported'
+    cm.save!
+    Canvas::Migration::Worker::CCWorker.expects(:new).never
+    cm.queue_migration
+
+    ContentMigration.any_instance.expects(:import_content).never
+    Timecop.travel(50.hours.from_now) do
+      run_jobs
+    end
+
+    cm.reload
+    expect(cm).to be_failed
+    expect(cm.migration_issues).not_to be_empty
+    expect(cm.migration_issues.last.error_report.message).to include 'job expired'
+  end
 end