remove messages older than 360 days

fixes CNVS-9304

this commit also replaces the current MessageScrubber
class and moves its old function to the aptly named
DelayedMessageScrubber.

test plan:
  * in a rails console, run the following in a rails
    console:

  Setting.set('message_scrubber_limit', 7)
  scrubber = MessageScrubber.new
  scrubber.scrub_all

  * verify that all messages older than 7 days
    have been deleted and that messages newser than
    7 days (assuming you have any) still exist. you
    can do this by running the following in a rails
    console:

  # this should be 0
  Message.where('sent_at < ?', scrubber.limit).count

  # this should be > 0
  Message.where('sent_at > ?', scrubber.limit).count

Change-Id: I6c054971b3bcc8ec5a436edf4454ff0f0952f41b
Reviewed-on: https://gerrit.instructure.com/26017
Reviewed-by: Zach Pendleton <zachp@instructure.com>
Tested-by: Jenkins <jenkins@instructure.com>
QA-Review: Steven Shepherd <sshepherd@instructure.com>
Product-Review: Zach Pendleton <zachp@instructure.com>

config/periodic_jobs.rb

@@ -116,6 +116,13 @@ Delayed::Periodic.cron 'MessageScrubber.scrub_all', '0 0 * * *' do
   scrubber.scrub_all
 end
 
+Delayed::Periodic.cron 'DelayedMessageScrubber.scrub_all', '0 1 * * *' do
+  scrubber = DelayedMessageScrubber.new
+  scrubber.scrub_all
+end
+
+
+
 Dir[Rails.root.join('vendor', 'plugins', '*', 'config', 'periodic_jobs.rb')].each do |plugin_periodic_jobs|
   require plugin_periodic_jobs
 end

lib/delayed_message_scrubber.rb

@@ -0,0 +1,39 @@
+#
+# Copyright (C) 2013 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Public: Delete old (> 90 days) records from delayed_messages table.
+class DelayedMessageScrubber < MessageScrubber
+
+  protected
+
+  def filter_attribute
+    'send_at'
+  end
+
+  def klass
+    DelayedMessage
+  end
+
+  def limit_setting
+    'delayed_message_scrubber_limit'
+  end
+
+  def limit_size
+    90
+  end
+end

lib/message_scrubber.rb

@@ -16,11 +16,15 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# Public: Delete old (> 90 days) records from delayed_messages.
+# Public: Delete old (> 360 days) records from messages table.
 class MessageScrubber
+
   # Public: The minimum wait time in seconds between processing batches.
   MIN_DELAY = 1
 
+  # Public: The default batch size.
+  BATCH_SIZE = 1000
+
   attr_reader :batch_size, :delay, :limit, :logger
 
   # Public: Create a new MessageScrubber.
@@ -30,8 +34,8 @@ class MessageScrubber
   #   - delay: The delay, in seconds, between batches (default: 1).
   #   - logger: A logger object to log messages to (default: Rails.logger).
   def initialize(options = {})
-    @batch_size = options.fetch(:batch_size, 1000)
-    @limit      = Integer(Setting.get('message_scrubber_limit', 90)).days.ago
+    @batch_size = options.fetch(:batch_size, BATCH_SIZE)
+    @limit      = Integer(Setting.get(limit_setting, limit_size)).days.ago
     @delay      = options.fetch(:delay, MIN_DELAY)
     @logger     = options.fetch(:logger, Rails.logger)
   end
@@ -44,7 +48,7 @@ class MessageScrubber
   # Returns nothing.
   def scrub(options = {})
     dry_run = options.fetch(:dry_run, false)
-    scope   = DelayedMessage.where('send_at < ?', limit)
+    scope   = klass.where("#{filter_attribute} < ?", limit)
     dry_run ? log(scope) : delete_messages(scope)
   end
 
@@ -58,7 +62,7 @@ class MessageScrubber
     Shard.with_each_shard { scrub(options) }
   end
 
-  private
+  protected
 
   # Internal: Delete the current batch of messages.
   #
@@ -77,12 +81,40 @@ class MessageScrubber
     total
   end
 
+  # Internal: The column name to filter messages on (e.g. 'sent_at').
+  #
+  # Returns a column name string.
+  def filter_attribute
+    'sent_at'
+  end
+
+  # Internal: The class object to delete records from (e.g. 'Message').
+  #
+  # Returns class object.
+  def klass
+    Message
+  end
+
+  # Internal: The name of the Canvas setting this class' limit is stored in.
+  #
+  # Returns a setting name string.
+  def limit_setting
+    'message_scrubber_limit'
+  end
+
+  # Internal: The default limit (in days) to delete messages after.
+  #
+  # Returns a setting name string.
+  def limit_size
+    360
+  end
+
   # Internal: Log expected action.
   #
   # scope - The ActiveRecord scope to log.
   #
   # Returns nothing.
   def log(scope)
-    logger.info("MessageScrubber: #{scope.count} records would be deleted (older than #{limit})")
+    logger.info("#{self.class.to_s}: #{scope.count} records would be deleted (older than #{limit})")
   end
 end

spec/lib/delayed_message_scrubber_spec.rb

@@ -0,0 +1,78 @@
+#
+# Copyright (C) 2013 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+
+# Helpers
+def delayed_message(send_at)
+  message = DelayedMessage.new(notification: @notification, context: @context,
+          communication_channel: @recipient.communication_channel)
+  message.send_at = send_at
+  message.save!
+  message
+end
+
+def old_messages(count = 2)
+  (1..count).map do
+    delayed_message(100.days.ago)
+  end
+end
+
+def new_messages(count = 2)
+  (1..count).map do
+    delayed_message(Time.now)
+  end
+end
+
+describe DelayedMessageScrubber do
+
+  describe '#scrub' do
+
+    before(:each) do
+      @context      = course
+      @notification = Notification.create!(name: 'Test Notification', category: 'Test')
+      @recipient    = user
+
+      @recipient.communication_channels.create!(path_type: 'email', path: 'user@example.com')
+    end
+
+    it 'should delete delayed messages older than 90 days' do
+      messages = old_messages(2)
+      scrubber = DelayedMessageScrubber.new
+      scrubber.scrub
+      DelayedMessage.where(id: messages.map(&:id)).count.should == 0
+    end
+
+    it 'should not delete messages younger than 90 days' do
+      messages = old_messages(1) + new_messages(1)
+
+      scrubber = DelayedMessageScrubber.new
+      scrubber.scrub
+      DelayedMessage.where(id: messages.map(&:id)).count.should == 1
+    end
+
+    it 'should log predicted results if passed dry_run=true' do
+      logger   = mock
+      messages = old_messages(2)
+      scrubber = DelayedMessageScrubber.new(logger: logger)
+
+      logger.expects(:info).with("DelayedMessageScrubber: 2 records would be deleted (older than #{scrubber.limit})")
+      scrubber.scrub(dry_run: true)
+    end
+  end
+end

spec/lib/message_scrubber_spec.rb

@@ -19,23 +19,23 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
 
 # Helpers
-def delayed_message(send_at)
-  message = DelayedMessage.new(notification: @notification, context: @context,
+def message(sent_at)
+  message = Message.new(notification: @notification, context: @context,
           communication_channel: @recipient.communication_channel)
-  message.send_at = send_at
+  message.sent_at = sent_at
   message.save!
   message
 end
 
 def old_messages(count = 2)
   (1..count).map do
-    delayed_message(100.days.ago)
+    message(700.days.ago)
   end
 end
 
 def new_messages(count = 2)
   (1..count).map do
-    delayed_message(Time.now)
+    message(Time.now)
   end
 end
 
@@ -51,19 +51,19 @@ describe MessageScrubber do
       @recipient.communication_channels.create!(path_type: 'email', path: 'user@example.com')
     end
 
-    it 'should delete delayed messages older than 90 days' do
+    it 'should delete delayed messages older than 360 days' do
       messages = old_messages(2)
       scrubber = MessageScrubber.new
       scrubber.scrub
-      DelayedMessage.where(id: messages.map(&:id)).count.should == 0
+      Message.where(id: messages.map(&:id)).count.should == 0
     end
 
-    it 'should not delete messages younger than 90 days' do
+    it 'should not delete messages younger than 360 days' do
       messages = old_messages(1) + new_messages(1)
 
       scrubber = MessageScrubber.new
       scrubber.scrub
-      DelayedMessage.where(id: messages.map(&:id)).count.should == 1
+      Message.where(id: messages.map(&:id)).count.should == 1
     end
 
     it 'should log predicted results if passed dry_run=true' do
@@ -104,7 +104,7 @@ describe MessageScrubber do
       scrubber.scrub_all
       [@shard1, @shard2].each do |shard|
         shard.activate do
-          DelayedMessage.where(id: @messages.map(&:id)).count.should == 0
+          Message.where(id: @messages.map(&:id)).count.should == 0
         end
       end
     end