create enrollment_states table

creates a new table that will store the enrollment
state_based_on_date information that we've been previously
calculating over and over again all the time

we'll invalidate the rows whenever we make any date
changes and recalculate them in a delayed job

also we'll periodically recalculate as relevant dates pass
for the enrollments

if the time between when we invalidate the data and when
we recalculate it is acceptably short, then we should start
using it for actual database scoping for enrollments

test plan:

* regression test term/course/section dates for future, active
 and soft-concluded enrollments
* also regression test visibility access settings (e.g.
 "Restrict students from viewing course before start date")
 on accounts and courses

closes #CNVS-29460

Change-Id: I27d23c0eb49e5b1b9a6fe532409e51be8a07be52
Reviewed-on: https://gerrit.instructure.com/80000
QA-Review: Deepeeca Soundarrajan <dsoundarrajan@instructure.com>
Tested-by: Jenkins
Reviewed-by: Jeremy Stanley <jeremy@instructure.com>
Product-Review: James Williams  <jamesw@instructure.com>

app/models/account.rb

@@ -221,7 +221,6 @@ class Account < ActiveRecord::Base
   end
 
   def settings=(hash)
-    @invalidate_settings_cache = true
     if hash.is_a?(Hash)
       hash.each do |key, val|
         if account_settings_options && account_settings_options[key.to_sym]
@@ -409,7 +408,10 @@ class Account < ActiveRecord::Base
 
   def settings
     result = self.read_attribute(:settings)
-    return result if result
+    if result
+      @old_settings ||= result.dup
+      return result
+    end
     return write_attribute(:settings, {}) unless frozen?
     {}.freeze
   end
@@ -544,7 +546,12 @@ class Account < ActiveRecord::Base
 
   def invalidate_caches_if_changed
     @invalidations ||= []
-    @invalidations += Account.inheritable_settings if @invalidate_settings_cache
+    if @old_settings
+      Account.inheritable_settings.each do |key|
+        @invalidations << key if @old_settings[key] != settings[key] # only invalidate if needed
+      end
+      @old_settings = nil
+    end
     if @invalidations.present?
       shard.activate do
         @invalidations.each do |key|
@@ -564,6 +571,11 @@ class Account < ActiveRecord::Base
           Rails.cache.delete([key, global_id].cache_key)
         end
       end
+
+      access_keys = keys & [:restrict_student_future_view, :restrict_student_past_view]
+      if access_keys.any?
+        EnrollmentState.invalidate_access_for_accounts([parent_account.id] + account_ids, access_keys)
+      end
     end
   end
 

app/models/course.rb

@@ -218,6 +218,7 @@ class Course < ActiveRecord::Base
   before_save :update_show_total_grade_as_on_weighting_scheme_change
   after_save :update_final_scores_on_weighting_scheme_change
   after_save :update_account_associations_if_changed
+  after_save :update_enrollment_states_if_necessary
   after_save :set_self_enrollment_code
 
   before_save :touch_root_folder_if_necessary
@@ -289,6 +290,24 @@ class Course < ActiveRecord::Base
     end
   end
 
+  def update_enrollment_states_if_necessary
+    if (changes.keys & %w{restrict_enrollments_to_course_dates account_id enrollment_term_id}).any? ||
+        (self.restrict_enrollments_to_course_dates? && (changes.keys & %w{start_at conclude_at}).any?) ||
+        (self.workflow_state_changed? && (completed? || self.workflow_state_was == 'completed'))
+        # a lot of things can change the date logic here :/
+
+      EnrollmentState.send_later_if_production(:invalidate_states_for_course_or_section, self) if self.enrollments.exists?
+      # if the course date settings have been changed, we'll end up reprocessing all the access values anyway, so no need to queue below for other setting changes
+    elsif @changed_settings
+      changed_keys = (@changed_settings & [:restrict_student_future_view, :restrict_student_past_view])
+      if changed_keys.any?
+        EnrollmentState.send_later_if_production(:invalidate_access_for_course, self, changed_keys)
+      end
+    end
+
+    @changed_settings = nil
+  end
+
   def module_based?
     Rails.cache.fetch(['module_based_course', self].cache_key) do
       self.context_modules.active.any?{|m| m.completion_requirements && !m.completion_requirements.empty? }
@@ -2505,7 +2524,12 @@ class Course < ActiveRecord::Base
         end
       end
       def #{setting}=(val)
-        settings_frd[#{setting.inspect}] = #{cast_expression}
+        new_val = #{cast_expression}
+        if settings_frd[#{setting.inspect}] != new_val
+          @changed_settings ||= []
+          @changed_settings << #{setting.inspect}
+          settings_frd[#{setting.inspect}] = new_val
+        end
       end
     CODE
     alias_method "#{setting}?", setting if opts[:boolean]

app/models/course_section.rb

@@ -47,6 +47,7 @@ class CourseSection < ActiveRecord::Base
   before_save :maybe_touch_all_enrollments
   after_save :update_account_associations_if_changed
   after_save :delete_enrollments_later_if_deleted
+  after_save :update_enrollment_states_if_necessary
 
   include StickySisFields
   are_sis_sticky :course_id, :name, :start_at, :end_at, :restrict_enrollments_to_section_dates
@@ -271,4 +272,10 @@ class CourseSection < ActiveRecord::Base
   def common_to_users?(users)
     users.all?{ |user| self.student_enrollments.active.for_user(user).count > 0 }
   end
+
+  def update_enrollment_states_if_necessary
+    if self.restrict_enrollments_to_section_dates_changed? || (self.restrict_enrollments_to_section_dates? && (changes.keys & %w{start_at end_at}).any?)
+      EnrollmentState.send_later_if_production(:invalidate_states_for_course_or_section, self)
+    end
+  end
 end

app/models/enrollment.rb

@@ -39,6 +39,8 @@ class Enrollment < ActiveRecord::Base
   belongs_to :role
   include Role::AssociationHelper
 
+  has_one :enrollment_state, :dependent => :destroy
+
   has_many :role_overrides, :as => :context
   has_many :pseudonyms, :primary_key => :user_id, :foreign_key => :user_id
   has_many :course_account_associations, :foreign_key => 'course_id', :primary_key => 'course_id'
@@ -69,9 +71,10 @@ class Enrollment < ActiveRecord::Base
   after_save :reset_notifications_cache
   after_save :update_assignment_overrides_if_needed
   after_save :dispatch_invitations_later
+  after_save :recalculate_enrollment_state
   after_destroy :update_assignment_overrides_if_needed
 
-  attr_accessor :already_enrolled, :available_at, :soft_completed_at, :need_touch_user
+  attr_accessor :already_enrolled, :need_touch_user
   attr_accessible :user, :course, :workflow_state, :course_section, :limit_privileges_to_course_section, :already_enrolled, :start_at, :end_at
 
   scope :current, -> { joins(:course).where(QueryBuilder.new(:active).conditions).readonly(false) }
@@ -659,76 +662,44 @@ class Enrollment < ActiveRecord::Base
     @enrollment_dates
   end
 
-  def state_based_on_date
-    RequestCache.cache('enrollment_state_based_on_date', self, self.workflow_state) do
-      calculated_state_based_on_date
-    end
+  def enrollment_state
+    self.association(:enrollment_state).target ||=
+      self.shard.activate do
+        EnrollmentState.unique_constraint_retry do
+          EnrollmentState.where(:enrollment_id => self).first_or_create
+        end
+      end
+    super
   end
 
-  def calculated_state_based_on_date
-    if state == :completed || ([:invited, :active].include?(state) && self.course.completed?)
-      if self.restrict_past_view?
-        return :inactive
-      else
-        return :completed
-      end
+  def recalculate_enrollment_state
+    if (self.changes.keys & %w{workflow_state start_at end_at}).any?
+      self.enrollment_state.reload
+      self.enrollment_state.state_is_current = false
     end
+    self.enrollment_state.ensure_current_state
+  end
 
-    unless [:invited, :active].include?(state)
-      return state
-    end
-
-    ranges = self.enrollment_dates
-    now    = Time.now
-    ranges.each do |range|
-      start_at, end_at = range
-      # start_at <= now <= end_at, allowing for open ranges on either end
-      return state if (start_at || now) <= now && now <= (end_at || now)
-    end
-
-    # Not strictly within any range
-    return state unless global_start_at = ranges.map(&:compact).map(&:min).compact.min
-    if global_start_at < now
-      self.soft_completed_at = ranges.map(&:last).compact.min
-      self.restrict_past_view? ? :inactive : :completed
-    elsif self.fake_student? # Allow student view students to use the course before the term starts
-      state
-    else
-      self.available_at = global_start_at
-      if view_restrictable? && !self.restrict_future_view?
-        if state == :active
-          # an accepted enrollment state means they still can't participate yet,
-          # but should be able to view just like an invited enrollment
-          :accepted
-        else
-          state
-        end
-      else
-        :inactive
-      end
+  def state_based_on_date
+    RequestCache.cache('enrollment_state_based_on_date', self, self.workflow_state) do
+      self.enrollment_state.get_effective_state
     end
   end
 
   def readable_state_based_on_date
     # when view restrictions are in place, the effective state_based_on_date is :inactive, but
     # to admins we should show that they are :completed or :pending
+    self.enrollment_state.get_display_state
+  end
 
-    if state == :completed || ([:invited, :active].include?(state) && self.course.completed?)
-      :completed
-    else
-      date_state = state_based_on_date
-      if self.available_at
-        :pending
-      elsif self.soft_completed_at
-        :completed
-      else
-        date_state
-      end
+  def available_at
+    if self.enrollment_state.pending?
+      self.enrollment_state.state_valid_until
     end
   end
 
   def view_restrictable?
-    self.student? || self.observer?
+    (self.student? && !self.fake_student?) || self.observer?
   end
 
   def restrict_past_view?
@@ -764,8 +735,7 @@ class Enrollment < ActiveRecord::Base
   end
 
   def completed?
-    s = self.state_based_on_date
-    (s == :completed) || (s == :inactive && !!completed_at)
+    self.enrollment_state.get_display_state == :completed
   end
 
   def explicitly_completed?
@@ -773,7 +743,13 @@ class Enrollment < ActiveRecord::Base
   end
 
   def completed_at
-    self.read_attribute(:completed_at) || (self.state_based_on_date && self.soft_completed_at) # soft_completed_at is loaded through state_based_on_date
+    if date = self.read_attribute(:completed_at)
+      date
+    else
+      if completed?
+        self.enrollment_state.state_started_at
+      end
+    end
   end
 
   alias_method :destroy_permanently!, :destroy

app/models/enrollment_dates_override.rb

@@ -22,9 +22,11 @@ class EnrollmentDatesOverride < ActiveRecord::Base
 
   attr_accessible :context, :enrollment_type, :enrollment_term, :start_at, :end_at
 
-  before_save :touch_all_courses
+  after_save :update_courses_and_states_if_necessary
 
-  def touch_all_courses
-    self.enrollment_term.update_courses_later if self.changed?
+  def update_courses_and_states_if_necessary
+    if self.changed?
+      self.enrollment_term.update_courses_and_states_later(self.enrollment_type)
+    end
   end
 end

app/models/enrollment_state.rb

@@ -0,0 +1,234 @@
+class EnrollmentState < ActiveRecord::Base
+  belongs_to :enrollment
+
+  self.primary_key = 'enrollment_id'
+
+  def hash
+    global_enrollment_id.hash
+  end
+
+  def state_needs_recalculation?
+    !self.state_is_current? || self.state_valid_until && self.state_valid_until < Time.now
+  end
+
+  def ensure_current_state
+    self.recalculate_state if self.state_needs_recalculation?
+    self.recalculate_access if !self.access_is_current?
+    self.save! if self.changed?
+  end
+
+  def get_effective_state
+    self.ensure_current_state
+
+    if restricted_access?
+      :inactive
+    elsif self.state == 'pending_invited'
+      :invited
+    elsif self.state == 'pending_active'
+      :accepted
+    else
+      self.state.to_sym
+    end
+  end
+
+  def get_display_state
+    self.ensure_current_state
+
+    if pending?
+      :pending
+    else
+      self.state.to_sym
+    end
+  end
+
+  def pending?
+    %w{pending_active pending_invited}.include?(self.state)
+  end
+
+  def recalculate_state
+    self.state_valid_until = nil
+    self.state_started_at = nil
+
+    wf_state = self.enrollment.workflow_state
+    invited_or_active = %w{invited active}.include?(wf_state)
+
+    if invited_or_active
+      if self.enrollment.course.completed?
+        self.state = 'completed'
+      else
+        self.calculate_state_based_on_dates
+      end
+    else
+      self.state = wf_state
+    end
+    self.state_is_current = true
+
+    if self.state_changed? && self.enrollment.view_restrictable?
+      self.access_is_current = false
+    end
+
+    # TODO: remove when diagnostic columns are removed
+    self.state_recalculated_at = Time.now.utc
+  end
+
+  def calculate_state_based_on_dates
+    wf_state = self.enrollment.workflow_state
+    ranges = self.enrollment.enrollment_dates
+    now = Time.now
+
+    # start_at <= now <= end_at, allowing for open ranges on either end
+    if range = ranges.detect{|start_at, end_at| (start_at || now) <= now && now <= (end_at || now) }
+      self.state = wf_state
+      start_at, end_at = range
+      self.state_started_at = start_at
+      self.state_valid_until = end_at
+    else
+      global_start_at = ranges.map(&:compact).map(&:min).compact.min
+
+      if !global_start_at
+        # Not strictly within any range
+        self.state = wf_state
+      elsif global_start_at < now
+        self.state_started_at = ranges.map(&:last).compact.min
+        self.state = 'completed'
+      elsif self.enrollment.fake_student? # Allow student view students to use the course before the term starts
+        self.state = wf_state
+      else
+        self.state_valid_until = global_start_at
+        if self.enrollment.view_restrictable?
+          # these enrollment states mean they still can't participate yet even if they've accepted it,
+          # but should be able to view just like an invited enrollment
+          if wf_state == 'active'
+            self.state = 'pending_active'
+          else
+            self.state = 'pending_invited'
+          end
+        else
+          # admin user restricted by term dates
+          self.state = 'inactive'
+        end
+      end
+    end
+  end
+
+  def recalculate_access
+    if self.enrollment.view_restrictable?
+      self.restricted_access =
+        case self.state
+        when 'pending_invited', 'pending_active'
+          self.enrollment.restrict_future_view?
+        when 'completed'
+          self.enrollment.restrict_past_view?
+        else
+          false
+        end
+    else
+      self.restricted_access = false
+    end
+    self.access_is_current = true
+
+    # TODO: remove when diagnostic columns are removed
+    self.access_recalculated_at = Time.now.utc
+  end
+
+  def self.enrollments_needing_calculation(scope=Enrollment.all)
+    scope.joins("LEFT OUTER JOIN #{EnrollmentState.quoted_table_name} ON enrollment_states.enrollment_id = enrollments.id").
+      where("enrollment_states IS NULL OR enrollment_states.state_is_current = ? OR enrollment_states.access_is_current = ?", false, false)
+  end
+
+  def self.process_states_in_ranges(start_at, end_at, enrollment_scope=Enrollment.all)
+    Enrollment.find_ids_in_ranges(:start_at => start_at, :end_at => end_at, :batch_size => 250) do |min_id, max_id|
+      process_states_for(enrollments_needing_calculation(enrollment_scope).where(:id => min_id..max_id))
+    end
+  end
+
+  def self.process_term_states_in_ranges(start_at, end_at, term, enrollment_type=nil)
+    scope = term.enrollments
+    scope = scope.where(:type => enrollment_type) if enrollment_type
+    process_states_in_ranges(start_at, end_at, scope)
+  end
+
+  def self.process_account_states_in_ranges(start_at, end_at, account_ids)
+    process_states_in_ranges(start_at, end_at, enrollments_for_account_ids(account_ids))
+  end
+
+  def self.process_states_for(enrollments)
+    enrollments = Array(enrollments)
+    Canvas::Builders::EnrollmentDateBuilder.preload(enrollments, false)
+
+    enrollments.each do |enrollment|
+      update_enrollment(enrollment)
+    end
+  end
+
+  def self.update_enrollment(enrollment)
+    enrollment.enrollment_state.ensure_current_state
+  end
+
+  INVALIDATEABLE_STATES = %w{pending_invited pending_active invited active completed inactive}.freeze # don't worry about creation_pending or rejected, etc
+  def self.invalidate_states(enrollment_scope)
+    EnrollmentState.where(:enrollment_id => enrollment_scope, :state_is_current => true, :state => INVALIDATEABLE_STATES).update_all(:state_is_current => false, :state_invalidated_at => Time.now.utc)
+  end
+
+  def self.invalidate_access(enrollment_scope, states_to_update)
+    EnrollmentState.where(:enrollment_id => enrollment_scope, :access_is_current => true, :state => states_to_update).update_all(:access_is_current => false, :access_invalidated_at => Time.now.utc)
+  end
+
+  def self.enrollments_for_account_ids(account_ids)
+    Enrollment.joins(:course).where(:courses => {:account_id => account_ids}).where(:type => %w{StudentEnrollment ObserverEnrollment})
+  end
+
+  ENROLLMENT_BATCH_SIZE = 20_000
+
+  def self.invalidate_states_for_term(term, enrollment_type=nil)
+    # invalidate and re-queue individual jobs for reprocessing because it might be too big to do all at once
+    scope = term.enrollments
+    scope = scope.where(:type => enrollment_type) if enrollment_type
+    Enrollment.find_ids_in_ranges(:batch_size => ENROLLMENT_BATCH_SIZE) do |min_id, max_id|
+      if invalidate_states(scope.where(:id => min_id..max_id)) > 0
+        EnrollmentState.send_later_if_production_enqueue_args(:process_term_states_in_ranges, {:priority => Delayed::LOW_PRIORITY}, min_id, max_id, term, enrollment_type)
+      end
+    end
+  end
+
+  def self.invalidate_states_for_course_or_section(course_or_section)
+    scope = course_or_section.enrollments
+    if invalidate_states(scope) > 0
+      process_states_for(enrollments_needing_calculation(scope))
+    end
+  end
+
+  def self.access_states_to_update(changed_keys)
+    states_to_update = []
+    # only need to invalidate access for future students if future access changed, etc
+    states_to_update += ['pending_invited', 'pending_active'] if changed_keys.include?(:restrict_student_future_view)
+    states_to_update += ['completed'] if changed_keys.include?(:restrict_student_past_view)
+    states_to_update
+  end
+
+  def self.invalidate_access_for_accounts(account_ids, changed_keys)
+    states_to_update = access_states_to_update(changed_keys)
+    Enrollment.find_ids_in_ranges(:batch_size => ENROLLMENT_BATCH_SIZE) do |min_id, max_id|
+      scope = enrollments_for_account_ids(account_ids).where(:id => min_id..max_id)
+      if invalidate_access(scope, states_to_update) > 0
+        EnrollmentState.send_later_if_production_enqueue_args(:process_account_states_in_ranges, {:priority => Delayed::LOW_PRIORITY}, min_id, max_id, account_ids)
+      end
+    end
+  end
+
+  def self.invalidate_access_for_course(course, changed_keys)
+    states_to_update = access_states_to_update(changed_keys)
+    scope = course.enrollments.where(:type => %w{StudentEnrollment ObserverEnrollment})
+    if invalidate_access(scope, states_to_update) > 0
+      process_states_for(enrollments_needing_calculation(scope))
+    end
+  end
+
+  def self.recalculate_expired_states
+    while (enrollments = Enrollment.joins(:enrollment_state).where("enrollment_states.state_valid_until IS NOT NULL AND
+           enrollment_states.state_valid_until < ?", Time.now.utc).limit(250).to_a) && enrollments.any?
+      EnrollmentState.where(:enrollment_id => enrollments).update_all("state_invalidated_at = state_valid_until") # temporary, to determine how long it took to update
+      process_states_for(enrollments)
+    end
+  end
+end

app/models/enrollment_term.rb

@@ -34,7 +34,7 @@ class EnrollmentTerm < ActiveRecord::Base
   validate :check_if_deletable
 
   before_validation :verify_unique_sis_source_id
-  before_save :update_courses_later_if_necessary
+  after_save :update_courses_later_if_necessary
   before_update :destroy_orphaned_grading_period_group
 
   include StickySisFields
@@ -51,7 +51,9 @@ class EnrollmentTerm < ActiveRecord::Base
   end
 
   def update_courses_later_if_necessary
-    self.update_courses_later if !self.new_record? && (self.start_at_changed? || self.end_at_changed?)
+    if !self.new_record? && (self.start_at_changed? || self.end_at_changed?)
+      self.update_courses_and_states_later
+    end
   end
 
   # specifically for use in specs
@@ -60,13 +62,16 @@ class EnrollmentTerm < ActiveRecord::Base
   end
 
   def touch_all_courses
-    return if new_record?
     self.courses.touch_all
   end
 
-  def update_courses_later
+  def update_courses_and_states_later(enrollment_type=nil)
+    return if new_record?
+
     self.send_later_if_production(:touch_all_courses) unless @touched_courses
     @touched_courses = true
+
+    EnrollmentState.send_later_if_production(:invalidate_states_for_term, self, enrollment_type)
   end
 
   def self.i18n_default_term_name

config/initializers/periodic_jobs.rb

@@ -158,4 +158,8 @@ Rails.configuration.after_initialize do
                                   singleton: 'AccountAuthorizationConfig::SAML::InCommon.refresh_providers')
     end
   end
+
+  Delayed::Periodic.cron 'EnrollmentState.recalculate_expired_states', '*/5 * * * *', :priority => Delayed::LOW_PRIORITY do
+    with_each_shard_by_database(EnrollmentState, :recalculate_expired_states)
+  end
 end

db/migrate/20141212134557_change_enrollments_role_id_null.rb

@@ -25,7 +25,7 @@ class ChangeEnrollmentsRoleIdNull < ActiveRecord::Migration
 
     # delete remaining duplicate enrollments
     Enrollment.find_ids_in_ranges(batch_size: 10000) do |start_id, end_id|
-      Enrollment.where(id: start_id..end_id, type: type, role_id: nil).delete_all
+      Enrollment.where(id: start_id..end_id, type: type, role_id: nil).each(&:destroy_permanently!)
     end
   end
 

db/migrate/20160510152226_create_enrollment_states.rb

@@ -0,0 +1,36 @@
+class CreateEnrollmentStates < ActiveRecord::Migration
+  tag :predeploy
+
+  def up
+    create_table :enrollment_states, :id => false do |t|
+      t.integer :enrollment_id, limit: 8, null: false
+
+      t.string :state
+      t.boolean :state_is_current, null: false, default: false
+      t.datetime :state_started_at
+      t.datetime :state_valid_until
+
+      t.boolean :restricted_access, null: false, default: false
+      t.boolean :access_is_current, null: false, default: false
+
+      # these will go away - for initial diagnostic purposes
+      t.datetime :state_invalidated_at
+      t.datetime :state_recalculated_at
+      t.datetime :access_invalidated_at
+      t.datetime :access_recalculated_at
+    end
+
+    add_index :enrollment_states, :enrollment_id, :unique => true, :name => "index_enrollment_states"
+    execute("ALTER TABLE #{EnrollmentState.quoted_table_name} ADD CONSTRAINT enrollment_states_pkey PRIMARY KEY USING INDEX index_enrollment_states")
+
+    add_index :enrollment_states, :state
+    add_index :enrollment_states, [:state_is_current, :access_is_current], :name => "index_enrollment_states_on_currents"
+    add_index :enrollment_states, :state_valid_until
+
+    add_foreign_key :enrollment_states, :enrollments
+  end
+
+  def down
+    drop_table :enrollment_states
+  end
+end

db/migrate/20160517153405_build_enrollment_states.rb

@@ -0,0 +1,36 @@
+class BuildEnrollmentStates < ActiveRecord::Migration
+  tag :postdeploy
+
+  def up
+    # try to partition off ranges of ids in the table with at most 10,000 ids per partition
+    ranges = []
+    current_min = Enrollment.minimum(:id)
+    return unless current_min
+
+    range_size = 10_000
+
+    while current_min
+      current_max = current_min + range_size - 1
+
+      next_min = Enrollment.where("id > ?", current_max).minimum(:id)
+      if next_min
+        ranges << [current_min, current_max]
+      elsif !next_min && ranges.any?
+        ranges << [current_min, nil]
+      end
+      current_min = next_min
+    end
+
+    unless ranges.any?
+      ranges = [[nil, nil]]
+    end
+
+    ranges.each do |start_at, end_at|
+      EnrollmentState.send_later_if_production_enqueue_args(:process_states_in_ranges,
+        {:strand => "enrollment_state_building_#{Shard.current.database_server.id}", :priority => Delayed::MAX_PRIORITY}, start_at, end_at)
+    end
+  end
+
+  def down
+  end
+end

lib/canvas/builders/enrollment_date_builder.rb

@@ -28,18 +28,29 @@ class EnrollmentDateBuilder
     @enrollment_dates = []
   end
 
-  def self.preload(enrollments)
+  def self.preload(enrollments, use_cache=true)
     return if enrollments.empty?
-    courses_loaded = enrollments.first.association(:course).loaded?
-    ActiveRecord::Associations::Preloader.new.preload(enrollments, :course)unless courses_loaded
+    preload_state(enrollments)
 
-    to_preload = enrollments.reject { |e| fetch(e) }
+    courses_loaded = enrollments.first.association(:course).loaded?
+    ActiveRecord::Associations::Preloader.new.preload(enrollments, :course) unless courses_loaded
+
+    to_preload = use_cache ? enrollments.reject { |e| fetch(e) } : enrollments
     return if to_preload.empty?
     ActiveRecord::Associations::Preloader.new.preload(to_preload, :course_section)
     ActiveRecord::Associations::Preloader.new.preload(to_preload.map(&:course).uniq, :enrollment_term)
     to_preload.each { |e| build(e) }
   end
 
+  # TODO: other places where we use #preload should be replaced with #preload_state after all the states are created
+  def self.preload_state(enrollments)
+    return if enrollments.empty?
+
+    unless enrollments.first.association(:enrollment_state).loaded?
+      ActiveRecord::Associations::Preloader.new.preload(enrollments, :enrollment_state)
+    end
+  end
+
   def self.cache_key(enrollment)
     [enrollment, enrollment.course, 'enrollment_date_ranges'].cache_key
   end

lib/data_fixup/syckness_cleanser.rb

@@ -3,6 +3,7 @@ module DataFixup
     def self.columns_hash
       result = ActiveRecord::Base.all_models.map do |model|
         next unless model.superclass == ActiveRecord::Base
+        next unless model.connection.table_exists?(model.table_name)
         next if model.name == 'RemoveQuizDataIds::QuizQuestionDataMigrationARShim'
 
         attributes = model.serialized_attributes.select do |attr, coder|

spec/apis/v1/accounts_api_spec.rb

@@ -483,7 +483,7 @@ describe "Accounts API", type: :request do
       Time.use_zone(@user.time_zone) do
         @me = @user
         @c1 = course_model(:name => 'c1', :account => @a1, :root_account => @a1)
-        @c1.enrollments.delete_all
+        @c1.enrollments.each(&:destroy_permanently!)
         @c2 = course_model(:name => 'c2', :account => @a2, :root_account => @a1, :sis_source_id => 'sis2')
         @c2.course_sections.create!
         @c2.course_sections.create!

spec/apis/v1/assignment_groups_api_spec.rb

@@ -232,7 +232,7 @@ describe AssignmentGroupsController, type: :request do
     it "should only return visible assignments when differentiated assignments is on" do
       setup_groups
       setup_four_assignments(only_visible_to_overrides: true)
-      @user.enrollments.each(&:delete)
+      @user.enrollments.each(&:destroy_permanently!)
       @section = @course.course_sections.create!(name: "test section")
       student_in_section(@section, user: @user)
       # make a1 and a3 visible

spec/apis/v1/courses_api_spec.rb

@@ -1202,10 +1202,10 @@ describe CoursesController, type: :request do
       end
 
       it "should deal gracefully with an invalid course id" do
-        @course2.enrollments.scope.delete_all
+        @course2.enrollments.each(&:destroy_permanently!)
         @course2.course_account_associations.scope.delete_all
         @course2.course_sections.scope.delete_all
-        @course2.destroy_permanently!
+        @course2.reload.destroy_permanently!
         json = api_call(:put, @path + "?event=offer&course_ids[]=#{@course1.id}&course_ids[]=#{@course2.id}",
                         @params.merge(:event => 'offer', :course_ids => [@course1.id.to_s, @course2.id.to_s]))
         run_jobs
@@ -1264,10 +1264,10 @@ describe CoursesController, type: :request do
       end
 
       it "should report a failure if no updates succeeded" do
-        @course2.enrollments.scope.delete_all
+        @course2.enrollments.each(&:destroy_permanently!)
         @course2.course_account_associations.scope.delete_all
         @course2.course_sections.scope.delete_all
-        @course2.destroy_permanently!
+        @course2.reload.destroy_permanently!
         json = api_call(:put, @path + "?event=offer&course_ids[]=#{@course2.id}",
                         @params.merge(:event => 'offer', :course_ids => [@course2.id.to_s]))
         run_jobs

spec/lib/dates_overridable_spec.rb

@@ -91,7 +91,7 @@ shared_examples_for "an object whose dates are overridable" do
 
         expect(overridable.overrides_for(@student, ensure_set_not_empty: true).size).to eq 1
 
-        override_student.user.enrollments.delete_all
+        override_student.user.enrollments.each(&:destroy_permanently!)
 
         expect(overridable.overrides_for(@student, ensure_set_not_empty: true)).to be_empty
       end

spec/migrations/link_missing_sis_observer_enrollments_spec.rb

@@ -13,7 +13,7 @@ describe 'DataFixup::LinkMissingSisObserverEnrollments' do
 
     observer.reload
     expect(observer.observer_enrollments.count).to eq 1
-    observer.enrollments.delete_all
+    observer.enrollments.each(&:destroy_permanently!)
 
     DataFixup::LinkMissingSisObserverEnrollments.run
 

spec/models/assignment_override_student_spec.rb

@@ -109,7 +109,7 @@ describe AssignmentOverrideStudent do
     it "if callbacks arent run clean_up_for_assignment should delete invalid overrides" do
       adhoc_override_with_student
       #no callbacks
-      @user.enrollments.delete_all
+      @user.enrollments.each(&:destroy_permanently!)
 
       expect(@ao.workflow_state).to eq("active")
       AssignmentOverrideStudent.clean_up_for_assignment(@assignment)

spec/models/enrollment_spec.rb

@@ -520,8 +520,7 @@ describe Enrollment do
         @enrollment.workflow_state = 'invited'
         @enrollment.save!
         expect(@enrollment.state).to eql(:invited)
-        @enrollment.accept
-        expect(@enrollment.reload.state).to eql(:active)
+        @enrollment.accept if @enrollment.invited?
         expect(@enrollment.state_based_on_date).to eql(state_based_state)
 
         @course.start_at = 2.days.from_now
@@ -552,6 +551,7 @@ describe Enrollment do
         @enrollment.workflow_state = 'invited'
         @enrollment.save!
         expect(@enrollment.state).to eql(:invited)
+        expect(@enrollment.state_based_on_date).to eql(:invited)
         @enrollment.accept
         expect(@enrollment.reload.state).to eql(:active)
         expect(@enrollment.state_based_on_date).to eql(:active)
@@ -562,19 +562,17 @@ describe Enrollment do
         @enrollment.workflow_state = 'invited'
         @enrollment.save!
         expect(@enrollment.state).to eql(:invited)
-        @enrollment.accept
-        expect(@enrollment.reload.state).to eql(:active)
         expect(@enrollment.state_based_on_date).to eql(:completed)
+        expect(@enrollment.accept).to be_falsey
 
         @term.start_at = 2.days.from_now
         @term.end_at = 4.days.from_now
         @term.save!
-        @enrollment.workflow_state = 'invited'
-        @enrollment.save!
         @enrollment.reload
         expect(@enrollment.state).to eql(:invited)
         expect(@enrollment.state_based_on_date).to eql(:invited)
         expect(@enrollment.accept).to be_truthy
+        expect(@enrollment.reload.state_based_on_date).to eql(@enrollment.admin? ? :active : :accepted)
       end
 
       def enrollment_dates_override_test
@@ -598,8 +596,6 @@ describe Enrollment do
         @enrollment.workflow_state = 'invited'
         @enrollment.save!
         expect(@enrollment.state).to eql(:invited)
-        @enrollment.accept
-        expect(@enrollment.reload.state).to eql(:active)
         expect(@enrollment.state_based_on_date).to eql(:completed)
 
         @override.start_at = 2.days.from_now

spec/models/enrollment_state_spec.rb

@@ -0,0 +1,326 @@
+require_relative "../spec_helper"
+
+describe EnrollmentState do
+
+  describe "#enrollments_needing_calculation" do
+    it "should find enrollments that don't have enrollment states (yet) as well" do
+      course
+      normal_enroll = student_in_course(:course => @course)
+
+      invalidated_enroll1 = student_in_course(:course => @course)
+      EnrollmentState.where(:enrollment_id => invalidated_enroll1).update_all(:state_is_current => false)
+      invalidated_enroll2 = student_in_course(:course => @course)
+      EnrollmentState.where(:enrollment_id => invalidated_enroll2).update_all(:access_is_current => false)
+
+      missing_enroll = student_in_course(:course => @course)
+      EnrollmentState.where(:enrollment_id => missing_enroll).delete_all
+
+      expect(EnrollmentState.enrollments_needing_calculation.to_a).to match_array([invalidated_enroll1, invalidated_enroll2, missing_enroll])
+    end
+
+    it "should be able to use a scope" do
+      course
+      enroll = student_in_course(:course => @course)
+      EnrollmentState.where(:enrollment_id => enroll).delete_all
+
+      expect(EnrollmentState.enrollments_needing_calculation(Enrollment.where.not(:id => nil)).to_a).to eq [enroll]
+      expect(EnrollmentState.enrollments_needing_calculation(Enrollment.where(:id => nil)).to_a).to be_empty
+    end
+  end
+
+  describe "#process_states_for" do
+    before :once do
+      course(:active_all => true)
+      @enrollment = student_in_course(:course => @course)
+    end
+
+    it "should create missing states" do
+      EnrollmentState.where(:enrollment_id => @enrollment).delete_all
+
+      @enrollment.reload
+
+      EnrollmentState.process_states_for(@enrollment)
+
+      @enrollment.reload
+      expect(@enrollment.enrollment_state).to be_present
+      expect(@enrollment.enrollment_state.state).to eq 'invited'
+    end
+
+    it "should reprocess invalidated states" do
+      EnrollmentState.where(:enrollment_id => @enrollment).update_all(:state_is_current => false, :state => "somethingelse")
+
+      @enrollment.reload
+      EnrollmentState.process_states_for(@enrollment)
+
+      @enrollment.reload
+      expect(@enrollment.enrollment_state.state_is_current?).to be_truthy
+      expect(@enrollment.enrollment_state.state).to eq 'invited'
+    end
+
+    it "should reprocess invalidated accesses" do
+      EnrollmentState.where(:enrollment_id => @enrollment).update_all(:access_is_current => false, :restricted_access => true)
+
+      @enrollment.reload
+      EnrollmentState.process_states_for(@enrollment)
+
+      @enrollment.reload
+      expect(@enrollment.enrollment_state.access_is_current?).to be_truthy
+      expect(@enrollment.enrollment_state.restricted_access?).to be_falsey
+    end
+  end
+
+  describe "state invalidation" do
+    it "should invalidate enrollments after enrollment term date change" do
+      course(:active_all => true)
+      other_enroll = student_in_course(:course => @course)
+
+      term = Account.default.enrollment_terms.create!
+      course(:active_all => true)
+      @course.enrollment_term = term
+      @course.save!
+      term_enroll = student_in_course(:course => @course)
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e != other_enroll}
+
+      term.reload
+      end_at = 2.days.ago
+      term.end_at = end_at
+      term.save!
+
+      term_enroll.reload
+      expect(term_enroll.enrollment_state.state_is_current?).to be_falsey
+
+      other_enroll.reload
+      expect(other_enroll.enrollment_state.state_is_current?).to be_truthy
+
+      state = term_enroll.enrollment_state
+      state.ensure_current_state
+      expect(state.state).to eq "completed"
+      expect(state.state_started_at).to eq end_at
+    end
+
+    it "should invalidate enrollments after enrollment term role-specific date change" do
+      term = Account.default.enrollment_terms.create!
+      course(:active_all => true)
+      @course.enrollment_term = term
+      @course.save!
+      other_enroll = teacher_in_course(:course => @course)
+      term_enroll = student_in_course(:course => @course)
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e == term_enroll}
+
+      override = term.enrollment_dates_overrides.new(:enrollment_type => "StudentEnrollment", :enrollment_term => term)
+      start_at = 2.days.from_now
+      override.start_at = start_at
+      override.save!
+
+      term_enroll.reload
+      expect(term_enroll.enrollment_state.state_is_current?).to be_falsey
+
+      other_enroll.reload
+      expect(other_enroll.enrollment_state.state_is_current?).to be_truthy
+
+      state = term_enroll.enrollment_state
+      state.ensure_current_state
+      expect(state.state).to eq "pending_invited"
+      expect(state.state_valid_until).to eq start_at
+    end
+
+    it "should invalidate enrollments after course date changes" do
+      course(:active_all => true)
+      @course.restrict_enrollments_to_course_dates = true
+      @course.save!
+      enroll = student_in_course(:course => @course)
+      enroll_state = enroll.enrollment_state
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e.course == @course}
+
+      @course.start_at = 4.days.ago
+      ended_at = 3.days.ago
+      @course.conclude_at = ended_at
+      @course.save!
+
+      enroll_state.reload
+      expect(enroll_state.state_is_current?).to be_falsey
+
+      enroll_state.ensure_current_state
+      expect(enroll_state.state).to eq 'completed'
+      expect(enroll_state.state_started_at).to eq ended_at
+    end
+
+    it "should invalidate enrollments after changing course setting overriding term dates" do
+      course(:active_all => true)
+      enroll = student_in_course(:course => @course)
+      enroll_state = enroll.enrollment_state
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e.course == @course}
+
+      @course.start_at = 4.days.ago
+      ended_at = 3.days.ago
+      @course.conclude_at = ended_at
+      @course.save!
+
+      # should not have changed yet - not overriding term dates
+      expect(enroll_state.state_is_current?).to be_truthy
+
+      @course.restrict_enrollments_to_course_dates = true
+      @course.save!
+
+      enroll_state.reload
+      expect(enroll_state.state_is_current?).to be_falsey
+
+      enroll_state.ensure_current_state
+      expect(enroll_state.state).to eq 'completed'
+      expect(enroll_state.state_started_at).to eq ended_at
+    end
+
+    it "should invalidate enrollments after changing course section dates" do
+      course(:active_all => true)
+      other_enroll = student_in_course(:course => @course)
+
+      section = @course.course_sections.create!
+      enroll = student_in_course(:course => @course, :section => section)
+      enroll_state = enroll.enrollment_state
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e.course_section == section}
+
+      section.restrict_enrollments_to_section_dates = true
+      section.save!
+      start_at = 1.day.from_now
+      section.start_at = start_at
+      section.save!
+
+      other_enroll.reload
+      expect(other_enroll.enrollment_state.state_is_current?).to be_truthy
+
+      enroll_state.reload
+      expect(enroll_state.state_is_current?).to be_falsey
+
+      enroll_state.ensure_current_state
+      expect(enroll_state.state).to eq 'pending_invited'
+      expect(enroll_state.state_valid_until).to eq start_at
+    end
+  end
+
+  describe "access invalidation" do
+    def restrict_view(account, type)
+      account.settings[type] = {:value => true, :locked => false}
+      account.save!
+    end
+
+    it "should invalidate access for future students when account future access settings are changed" do
+      course(:active_all => true)
+      other_enroll = student_in_course(:course => @course)
+      other_state = other_enroll.enrollment_state
+
+      future_enroll = student_in_course(:course => @course)
+      start_at = 2.days.from_now
+      future_enroll.start_at = start_at
+      future_enroll.end_at = 3.days.from_now
+      future_enroll.save!
+
+      future_state = future_enroll.enrollment_state
+      expect(future_state.state).to eq 'pending_invited'
+      expect(future_state.state_valid_until).to eq start_at
+      expect(future_state.restricted_access?).to be_falsey
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e != other_enroll}
+
+      restrict_view(Account.default, :restrict_student_future_view)
+
+      future_state.reload
+      expect(future_state.access_is_current).to be_falsey
+      other_state.reload
+      expect(other_state.access_is_current).to be_truthy
+
+      future_state.ensure_current_state
+      expect(future_state.restricted_access).to be_truthy
+      future_enroll.reload
+      expect(future_enroll).to be_inactive
+    end
+
+    it "should invalidate access for past students when past access settings are changed" do
+      course(:active_all => true)
+      other_enroll = student_in_course(:course => @course)
+      other_state = other_enroll.enrollment_state
+
+      sub_account = Account.default.sub_accounts.create!
+
+      course(:active_all => true, :account => sub_account)
+      @course.start_at = 3.days.ago
+      @course.conclude_at = 2.days.ago
+      @course.restrict_enrollments_to_course_dates = true
+      @course.save!
+      past_enroll = student_in_course(:course => @course)
+
+      past_state = past_enroll.enrollment_state
+      expect(past_state.state).to eq 'completed'
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e != other_enroll}
+
+      restrict_view(Account.default, :restrict_student_past_view)
+
+      past_state.reload
+      expect(past_state.access_is_current).to be_falsey
+      other_state.reload
+      expect(other_state.access_is_current).to be_truthy
+
+      past_state.ensure_current_state
+      expect(past_state.restricted_access).to be_truthy
+      past_enroll.reload
+      expect(past_enroll).to be_inactive
+    end
+
+    it "should invalidate access when course access settings change" do
+      course(:active_all => true)
+      @course.start_at = 3.days.from_now
+      @course.conclude_at = 4.days.from_now
+      @course.restrict_enrollments_to_course_dates = true
+      @course.save!
+      enroll = student_in_course(:course => @course)
+      enroll_state = enroll.enrollment_state
+
+      expect(enroll_state.state).to eq 'pending_invited'
+
+      EnrollmentState.expects(:update_enrollment).at_least_once.with {|e| e.course == @course}
+      @course.restrict_student_future_view = true
+      @course.save!
+
+      enroll_state.reload
+      expect(enroll_state.access_is_current).to be_falsey
+
+      enroll_state.ensure_current_state
+      expect(enroll_state.restricted_access).to be_truthy
+      enroll.reload
+      expect(enroll).to be_inactive
+    end
+  end
+
+  describe "#recalculate_expired_states" do
+    it "should recalculate expired states" do
+      course(:active_all => true)
+      @course.start_at = 3.days.from_now
+      end_at = 5.days.from_now
+      @course.conclude_at = end_at
+      @course.restrict_enrollments_to_course_dates = true
+      @course.save!
+
+      enroll = student_in_course(:course => @course)
+      enroll_state = enroll.enrollment_state
+      expect(enroll_state.state).to eq 'pending_invited'
+
+      Timecop.freeze(4.days.from_now) do
+        EnrollmentState.recalculate_expired_states
+        enroll_state.reload
+        expect(enroll_state.state).to eq 'invited'
+      end
+
+      Timecop.freeze(6.days.from_now) do
+        EnrollmentState.recalculate_expired_states
+        enroll_state.reload
+        expect(enroll_state.state).to eq 'completed'
+        expect(enroll_state.state_invalidated_at).to eq end_at # for diagnostic purposes
+      end
+    end
+  end
+end

spec/selenium/accounts_spec.rb

@@ -47,9 +47,9 @@ describe "account" do
     it "should be able to create a new course when no other courses exist" do
       Account.default.courses.each do |c|
         c.course_account_associations.scope.delete_all
-        c.enrollments.scope.delete_all
+        c.enrollments.each(&:destroy_permanently!)
         c.course_sections.scope.delete_all
-        c.destroy_permanently!
+        c.reload.destroy_permanently!
       end
 
       get "/accounts/#{Account.default.to_param}"

spec/selenium/helpers/assignments_common.rb

@@ -135,7 +135,7 @@ module AssignmentsCommon
     # 2 course sections, student in second section.
     @section1 = @course.course_sections.create!(:name => 'Section A')
     @section2 = @course.course_sections.create!(:name => 'Section B')
-    @course.student_enrollments.scope.delete_all # get rid of existing student enrollments, mess up section enrollment
+    @course.student_enrollments.each(&:destroy_permanently!) # get rid of existing student enrollments, mess up section enrollment
     # Overridden lock dates for 2nd section - different dates, but still in future
     @override = assignment_override_model(
       :assignment => @assignment,