recover as best we can when a job fails to fail, fixes #4951

Change-Id: I2be24518f337463f866f62789e6d1b8be6eac0e6
Reviewed-on: https://gerrit.instructure.com/4501
Reviewed-by: Bracken Mosbacker <bracken@instructure.com>
Tested-by: Hudson <hudson@instructure.com>

vendor/plugins/delayed_job/lib/delayed/backend/active_record.rb

@@ -127,6 +127,12 @@ module Delayed
             Failed.create(attrs)
             self.destroy
           end
+        rescue
+          # we got an error while failing the job -- we need to at least get
+          # the job out of the queue
+          self.destroy
+          # re-raise so the worker logs the error, at least
+          raise
         end
 
         # Get the current time (GMT or local depending on DB)

vendor/plugins/delayed_job/spec_canvas/job_spec.rb

@@ -21,4 +21,13 @@ describe Delayed::Job do
 
     lambda { [story, 1, story, false].send_later(:first) }.should raise_error
   end
+
+  it "should recover as well as possible from a failure failing a job" do
+    Delayed::Job::Failed.stub!(:create).and_return { raise "oh noes that was weird" }
+    job = "test".send_later :reverse
+    job_id = job.id
+    proc { job.fail! }.should raise_error
+    proc { Delayed::Job.find(job_id) }.should raise_error(ActiveRecord::RecordNotFound)
+    Delayed::Job.count.should == 0
+  end
 end