Retry file uploads for basic outcomes

fixes INTEROP-6674
flag=none

Test plan
- Have an LTI 1.1 tool set up that allows grade passback
- Send a request to https://<canvas>/api/lti/v1/tools/:tool_id/grade_passback
  with a bad file like this in your Canvas console or another with oauth
  installed:
  consumer = OAuth::Consumer.new(:tool.lti_key, :tool.lti_secret)
  client = OAuth::AccessToken.new(consumer)
  client.post(url, xml, 'Content-Type' => 'application/xml')
- The XML should look like below, but make sure to put in a url
  and a sourcedId from a real assignment/user in your instance
- Ensure the job attempts to run multiple times

<?xml version = "1.0" encoding = "UTF-8"?>
<imsx_POXEnvelopeRequest xmlns="http://www.imsglobal.org/services/ltiv1p1/xsd/imsoms_v1p0">
  <imsx_POXHeader>
    <imsx_POXRequestHeaderInfo>
      <imsx_version>V1.0</imsx_version>
      <imsx_messageIdentifier>999999123</imsx_messageIdentifier>
    </imsx_POXRequestHeaderInfo>
  </imsx_POXHeader>
  <imsx_POXBody>
    <replaceResultRequest>
      <resultRecord>
        <sourcedGUID>
          <sourcedId>3124567</sourcedId>
        </sourcedGUID>
        <result>
          <resultScore>
            <language>en</language>
            <textString>0.92</textString>
          </resultScore>
          <!-- Added element -->
          <resultData>
            <downloadUrl>(url)</downloadUrl>
            <documentName>(name)</documentName>
          </resultData>
        </result>
      </resultRecord>
    </replaceResultRequest>
  </imsx_POXBody>
</imsx_POXEnvelopeRequest>

Change-Id: I60977230b15b945e592225dfca0f5057e960f924
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/262457
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Xander Moffatt <xmoffatt@instructure.com>
Product-Review: Karl Lloyd <karl@instructure.com>
Reviewed-by: Xander Moffatt <xmoffatt@instructure.com>

lib/basic_lti/basic_outcomes.rb

@@ -35,6 +35,9 @@ module BasicLTI
       end
     end
 
+    # gives instfs about 7 hours to have an outage and eventually take the file
+    MAX_ATTEMPTS=10
+
     SOURCE_ID_REGEX = %r{^(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$}
 
     def self.decode_source_id(tool, sourceid)
@@ -273,25 +276,15 @@ to because the assignment has no points possible.
           self.code_major = 'failure'
           self.description = I18n.t('lib.basic_lti.no_points_possible', 'Assignment has no points possible.')
         else
+          create_homework_submission tool, submission_hash, assignment, user, new_score, raw_score, submitted_at_date
           if attachment
             job_options = {
               :priority => Delayed::HIGH_PRIORITY,
-              :max_attempts => 1,
+              :max_attempts => MAX_ATTEMPTS,
               :n_strand => Attachment.clone_url_strand(url)
             }
 
-            delay(**job_options).fetch_attachment_and_save_submission(
-              url,
-              attachment,
-              tool,
-              submission_hash,
-              assignment,
-              user,
-              new_score,
-              raw_score
-            )
-          else
-            create_homework_submission tool, submission_hash, assignment, user, new_score, raw_score, submitted_at_date
+            delay(**job_options).fetch_attachment(url, attachment)
           end
         end
 
@@ -352,9 +345,9 @@ to because the assignment has no points possible.
       end
 
       # rubocop:disable Metrics/ParameterLists
-      def fetch_attachment_and_save_submission(url, attachment, tool, submission_hash, assignment, user, new_score, raw_score)
+      def fetch_attachment(url, attachment)
         attachment.clone_url(url, 'rename', true)
-        create_homework_submission tool, submission_hash, assignment, user, new_score, raw_score
+        raise 'retry attachment clone' if attachment.file_state == 'errored'
       end
       # rubocop:enable Metrics/ParameterLists
 

spec/lib/basic_lti/basic_outcomes_spec.rb

@@ -18,7 +18,7 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper.rb')
+require 'spec_helper'
 
 require 'nokogiri'
 
@@ -529,6 +529,31 @@ describe BasicLTI::BasicOutcomes do
         expect(submission.reload.submission_type).to eq submission_type
       end
     end
+
+    context 'attachments' do
+      it 'retries attachments if they fail to upload' do
+        submission = assignment.submit_homework(
+          @user,
+          {
+            submission_type: "online_text_entry",
+            body: "sample text",
+            grade: "92%"
+          }
+        )
+        xml.css('resultScore').remove
+        xml.at_css('text').replace('<documentName>face.doc</documentName><downloadUrl>http://example.com/download</downloadUrl>')
+        BasicLTI::BasicOutcomes.process_request(tool, xml)
+        expect(Delayed::Job.strand_size('file_download/example.com')).to be > 0
+        Timecop.freeze do
+          run_jobs
+          expect(submission.reload.versions.count).to eq 2
+          expect(submission.attachments.count).to eq 1
+          expect(submission.attachments.first.file_state).to eq 'errored'
+          expect(Delayed::Job.strand_size('file_download/example.com')).to be > 0
+          expect(Delayed::Job.find_by(strand: 'file_download/example.com').run_at).to be > Time.zone.now + 5.seconds
+        end
+      end
+    end
   end
 
   describe "#process_request" do