batch up some massive updates at end of SIS import

fixes CNVS-7981

test plan:
 * run some SIS imports
 * they should succeed

Change-Id: Ia81981b04b3454d115ae607e9babed4c2548a906
Reviewed-on: https://gerrit.instructure.com/24092
Tested-by: Jenkins <jenkins@instructure.com>
Reviewed-by: Cody Cutrer <cody@instructure.com>
Product-Review: Cody Cutrer <cody@instructure.com>
QA-Review: Cody Cutrer <cody@instructure.com>

lib/sis/abstract_course_importer.rb

@@ -25,7 +25,9 @@ module SIS
       AbstractCourse.process_as_sis(@sis_options) do
         yield importer
       end
-      AbstractCourse.where(:id => importer.abstract_courses_to_update_sis_batch_id).update_all(:sis_batch_id => @batch_id) if @batch_id && !importer.abstract_courses_to_update_sis_batch_id.empty?
+      importer.abstract_courses_to_update_sis_batch_id.in_groups_of(1000, false) do |batch|
+        AbstractCourse.where(:id => batch).update_all(:sis_batch_id => @batch_id)
+      end if @batch_id
       @logger.debug("AbstractCourses took #{Time.now - start} seconds")
       return importer.success_count
     end

lib/sis/course_importer.rb

@@ -36,7 +36,9 @@ module SIS
       end
 
       Course.update_account_associations(course_ids_to_update_associations.to_a) unless course_ids_to_update_associations.empty?
-      Course.where(:id => courses_to_update_sis_batch_id).update_all(:sis_batch_id => @batch_id) if @batch_id && !courses_to_update_sis_batch_id.empty?
+      courses_to_update_sis_batch_id.in_groups_of(1000, false) do |batch|
+        Course.where(:id => batch).update_all(:sis_batch_id => @batch_id)
+      end if @batch_id
       @logger.debug("Courses took #{Time.now - start} seconds")
       return importer.success_count
     end

lib/sis/enrollment_importer.rb

@@ -36,15 +36,21 @@ module SIS
         end
       end
       @logger.debug("Raw enrollments took #{Time.now - start} seconds")
-      Enrollment.where(:id => i.enrollments_to_update_sis_batch_ids).update_all(:sis_batch_id => @batch_id) if @batch_id && !i.enrollments_to_update_sis_batch_ids.empty?
+      i.enrollments_to_update_sis_batch_ids.in_groups_of(1000, false) do |batch|
+        Enrollment.where(:id => batch).update_all(:sis_batch_id => @batch_id)
+      end if @batch_id
       # We batch these up at the end because we don't want to keep touching the same course over and over,
       # and to avoid hitting other callbacks for the course (especially broadcast_policy)
-      Course.where(:id => i.courses_to_touch_ids.to_a).update_all(:updated_at => Time.now.utc) unless i.courses_to_touch_ids.empty?
+      i.courses_to_touch_ids.to_a.in_groups_of(1000, false) do |batch|
+        Course.where(:id => batch).update_all(:updated_at => Time.now.utc)
+      end
       # We batch these up at the end because normally a user would get several enrollments, and there's no reason
       # to update their account associations on each one.
       i.incrementally_update_account_associations
       User.update_account_associations(i.update_account_association_user_ids.to_a, :account_chain_cache => i.account_chain_cache)
-      User.where(:id => i.users_to_touch_ids.to_a).update_all(:updated_at => Time.now.utc) unless i.users_to_touch_ids.empty?
+      i.users_to_touch_ids.to_a.in_groups_of(1000, false) do |batch|
+        User.where(:id => batch).update_all(:updated_at => Time.now.utc)
+      end
       @logger.debug("Enrollments with batch operations took #{Time.now - start} seconds")
       return i.success_count
     end

lib/sis/section_importer.rb

@@ -28,7 +28,9 @@ module SIS
         end
       end
       Course.update_account_associations(importer.course_ids_to_update_associations.to_a) unless importer.course_ids_to_update_associations.empty?
-      CourseSection.where(:id => importer.sections_to_update_sis_batch_ids).update_all(:sis_batch_id => @batch_id) if @batch_id && !importer.sections_to_update_sis_batch_ids.empty?
+      importer.sections_to_update_sis_batch_ids.in_groups_of(1000, false) do |batch|
+        CourseSection.where(:id => batch).update_all(:sis_batch_id => @batch_id)
+      end if @batch_id
       @logger.debug("Sections took #{Time.now - start} seconds")
       return importer.success_count
     end

lib/sis/user_importer.rb

@@ -34,7 +34,9 @@ module SIS
       end
       User.update_account_associations(importer.users_to_add_account_associations, :incremental => true, :precalculated_associations => {@root_account.id => 0})
       User.update_account_associations(importer.users_to_update_account_associations)
-      Pseudonym.where(:id => importer.pseudos_to_set_sis_batch_ids).update_all(:sis_batch_id => @batch_id) if @batch && !importer.pseudos_to_set_sis_batch_ids.empty?
+      importer.pseudos_to_set_sis_batch_ids.in_groups_of(1000, false) do |batch|
+        Pseudonym.where(:id => batch).update_all(:sis_batch_id => @batch_id)
+      end if @batch
       @logger.debug("Users took #{Time.now - start} seconds")
       return importer.success_count
     end