bounced email handling

fixes CNVS-15150

test plan
1. setup outgoing email through amazon ses test account
2. setup bounce_notifications.yml with sqs test creds
3. set a user's email address to 'bounce@simulator.amazonses.com'
4. cause three messages to be sent to that user
5. ensure that they are sent
6. wait for the bounce notification processor to run (~5 min)
7. casue another message to be sent to that user
8. ensure that message is not sent
9. repeat steps 3-8 using a different user and the address
 'suppressionlist@simulator.amazonses.com'
10. repeat steps 3-7 using a different user and the address
 'success@simulator.amazonses.com'
11. ensure that the fourth message is sent successfully
12. make sure no error reports or job failures are reported
 for the bounce notification processor

Change-Id: I060659c73a8b750c16f287e94f4198d8cb8633e5
Reviewed-on: https://gerrit.instructure.com/40254
Tested-by: Jenkins <jenkins@instructure.com>
Reviewed-by: Brad Horrocks <bhorrocks@instructure.com>
Reviewed-by: Cody Cutrer <cody@instructure.com>
QA-Review: Steven Shepherd <sshepherd@instructure.com>
Product-Review: Joel Hough <joel@instructure.com>
This commit is contained in:
Joel Hough 2014-08-28 19:45:03 -06:00
parent 0c1b95b4b3
commit d5dba5ffee
9 changed files with 315 additions and 39 deletions

View File

@ -0,0 +1,80 @@
#
# Copyright (C) 2014 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
class BounceNotificationProcessor
attr_reader :config
POLL_PARAMS = %w{initial_timeout idle_timeout wait_time_seconds visibility_timeout}.map(&:to_sym)
DEFAULT_CONFIG = {
bounce_queue_name: 'canvas_notifications_bounces',
idle_timeout: 10
}
def self.config
@@config ||= ConfigFile.load('bounce_notifications').try(:symbolize_keys)
end
def self.enabled?
!!config
end
def self.process(config = self.config)
self.new(config).process
end
def initialize(config = self.class.config)
@config = DEFAULT_CONFIG.merge(config)
end
def process
bounce_queue.poll(config.slice(*POLL_PARAMS)) do |message|
bounce_notification = parse_message(message)
process_bounce_notification(bounce_notification) if bounce_notification
end
end
private
def bounce_queue
return @bounce_queue if defined?(@bounce_queue)
sqs = AWS::SQS.new(access_key_id: config[:access_key_id], secret_access_key: config[:secret_access_key])
@bounce_queue = sqs.queues.named(config[:bounce_queue_name])
end
def parse_message(message)
sqs_body = JSON.parse(message.body)
sns_body = JSON.parse(sqs_body['Message'])
bounce_notification = sns_body['bounce']
end
def process_bounce_notification(bounce_notification)
return unless is_permanent_bounce?(bounce_notification)
bouncy_addresses(bounce_notification).each do |address|
CommunicationChannel.bounce_for_path(address)
end
end
def is_permanent_bounce?(bounce)
bounce['bounceType'] == 'Permanent'
end
def bouncy_addresses(bounce)
bounce['bouncedRecipients'].map {|r| r['emailAddress'] }
end
end

View File

@ -38,7 +38,7 @@ class CommunicationChannel < ActiveRecord::Base
EXPORTABLE_ASSOCIATIONS = [:pseudonyms, :pseudonym, :user]
before_save :consider_retiring, :assert_path_type, :set_confirmation_code
before_save :assert_path_type, :set_confirmation_code
before_save :consider_building_pseudonym
validates_presence_of :path, :path_type, :user, :workflow_state
validate :uniqueness_of_path
@ -59,7 +59,7 @@ class CommunicationChannel < ActiveRecord::Base
TYPE_FACEBOOK = 'facebook'
TYPE_PUSH = 'push'
RETIRE_THRESHOLD = 5
RETIRE_THRESHOLD = 3
def self.sms_carriers
@sms_carriers ||= Canvas::ICU.collate_by((ConfigFile.load('sms', false) ||
@ -300,11 +300,6 @@ class CommunicationChannel < ActiveRecord::Base
true
end
def consider_retiring
self.retire if self.bounce_count >= RETIRE_THRESHOLD
true
end
alias_method :destroy!, :destroy
def destroy
self.workflow_state = 'retired'
@ -324,9 +319,7 @@ class CommunicationChannel < ActiveRecord::Base
end
state :retired do
event :re_activate, :transitions_to => :active do
self.bounce_count = 0
end
event :re_activate, :transitions_to => :active
end
end
@ -388,4 +381,16 @@ class CommunicationChannel < ActiveRecord::Base
end
end
end
def bouncing?
self.bounce_count >= RETIRE_THRESHOLD
end
def self.bounce_for_path(path)
Shard.with_each_shard(CommunicationChannel.associated_shards(path)) do
CommunicationChannel.unretired.email.by_path(path).each do |channel|
channel.update_attribute(:bounce_count, channel.bounce_count + 1)
end
end
end
end

View File

@ -0,0 +1,7 @@
development:
access_key_id: access_key
secret_access_key: secret_key
# bounce_queue_name: canvas_notifications_bounces
# idle_timeout: 10
# You can also specify the following values to be passed into the sqs queue's
# poll command: initial_timeout, wait_time_seconds, visibility_timeout

View File

@ -123,7 +123,11 @@ Delayed::Periodic.cron 'DelayedMessageScrubber.scrub_all', '0 1 * * *' do
end
end
if BounceNotificationProcessor.enabled?
Delayed::Periodic.cron 'BounceNotificationProcessor.process', '*/5 * * * *' do
BounceNotificationProcessor.process
end
end
Dir[Rails.root.join('vendor', 'plugins', '*', 'config', 'periodic_jobs.rb')].each do |plugin_periodic_jobs|
require plugin_periodic_jobs

View File

@ -142,6 +142,7 @@ class NotificationMessageCreator
messages = []
message_options = message_options_for(user)
channels.reject!{ |channel| ['email', 'sms'].include?(channel.path_type) } if too_many_messages_for?(user) && @notification.summarizable?
channels.reject!(&:bouncing?)
channels.each do |channel|
messages << user.messages.build(message_options.merge(:communication_channel => channel,
:to => channel.path))

90
spec/fixtures/bounces.json vendored Normal file
View File

@ -0,0 +1,90 @@
[
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Permanent\",\"reportingMTA\":\"dsn; a14-1.smtp-out.amazonses.com\",\"bouncedRecipients\":[{\"status\":\"5.7.1\",\"action\":\"failed\",\"diagnosticCode\":\"smtp; 550-5.7.1 PERM_FAIL_POLICY Your email has been rejected because it violates\\n550 5.7.1 school policy. h9si40237239qgf.115 - gsmtp\",\"emailAddress\":\"hard@example.edu\"}],\"timestamp\":\"2014-08-22T12:25:46.786Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"source\":\"notifications@instructure.com\",\"timestamp\":\"2014-08-22T12:25:45.000Z\",\"messageId\":\"example-id\",\"destination\":[\"hard@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:25:46.854Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Permanent\",\"reportingMTA\":\"dsn; a14-1.smtp-out.amazonses.com\",\"bouncedRecipients\":[{\"status\":\"5.7.1\",\"action\":\"failed\",\"diagnosticCode\":\"smtp; 550-5.7.1 PERM_FAIL_POLICY Your email has been rejected because it violates\\n550 5.7.1 school policy. h9si40237239qgf.115 - gsmtp\",\"emailAddress\":\"hard@example.edu\"}],\"timestamp\":\"2014-08-22T12:25:46.786Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"source\":\"notifications@instructure.com\",\"timestamp\":\"2014-08-22T12:25:45.000Z\",\"messageId\":\"example-id\",\"destination\":[\"hard@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:25:46.854Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Permanent\",\"reportingMTA\":\"dsn; a14-1.smtp-out.amazonses.com\",\"bouncedRecipients\":[{\"status\":\"5.7.1\",\"action\":\"failed\",\"diagnosticCode\":\"smtp; 550-5.7.1 PERM_FAIL_POLICY Your email has been rejected because it violates\\n550 5.7.1 school policy. h9si40237239qgf.115 - gsmtp\",\"emailAddress\":\"hard@example.edu\"}],\"timestamp\":\"2014-08-22T12:25:46.786Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"source\":\"notifications@instructure.com\",\"timestamp\":\"2014-08-22T12:25:45.000Z\",\"messageId\":\"example-id\",\"destination\":[\"hard@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:25:46.854Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Permanent\",\"reportingMTA\":\"dsn; a14-1.smtp-out.amazonses.com\",\"bouncedRecipients\":[{\"status\":\"5.7.1\",\"action\":\"failed\",\"diagnosticCode\":\"smtp; 550-5.7.1 PERM_FAIL_POLICY Your email has been rejected because it violates\\n550 5.7.1 school policy. h9si40237239qgf.115 - gsmtp\",\"emailAddress\":\"hard@example.edu\"}],\"timestamp\":\"2014-08-22T12:25:46.786Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"source\":\"notifications@instructure.com\",\"timestamp\":\"2014-08-22T12:25:45.000Z\",\"messageId\":\"example-id\",\"destination\":[\"hard@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:25:46.854Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"Suppressed\",\"bounceType\":\"Permanent\",\"bouncedRecipients\":[{\"status\":\"5.1.1\",\"action\":\"failed\",\"diagnosticCode\":\"Amazon SES has suppressed sending to this address because it has a recent history of bouncing as an invalid address. For more information about how to remove an address from the suppression list, see the Amazon SES Developer Guide: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/remove-from-suppressionlist.html \",\"emailAddress\":\"hard@example.edu\"}],\"reportingMTA\":\"dns; amazonses.com\",\"timestamp\":\"2014-08-22T12:18:57.314Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"timestamp\":\"2014-08-22T12:18:57.000Z\",\"destination\":[\"hard@example.edu\"],\"source\":\"notifications@instructure.com\",\"messageId\":\"example-id\"}}",
"Timestamp" : "2014-08-22T12:18:57.429Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Transient\",\"reportingMTA\":\"dns;SCN-EX13MB03.scn.internal\",\"bouncedRecipients\":[{\"status\":\"5.7.1\",\"action\":\"failed\",\"diagnosticCode\":\"smtp;550 5.7.1 RESOLVER.RST.AuthRequired; authentication required\",\"emailAddress\":\"soft@example.edu\"}],\"timestamp\":\"2014-08-22T13:24:31.000Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"timestamp\":\"2014-08-22T13:24:23.000Z\",\"source\":\"notifications@instructure.com\",\"messageId\":\"example-id\",\"destination\":[\"soft@example.edu\"]}}",
"Timestamp" : "2014-08-22T13:24:36.599Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"Suppressed\",\"bounceType\":\"Permanent\",\"reportingMTA\":\"dns; amazonses.com\",\"bouncedRecipients\":[{\"status\":\"5.1.1\",\"action\":\"failed\",\"diagnosticCode\":\"Amazon SES has suppressed sending to this address because it has a recent history of bouncing as an invalid address. For more information about how to remove an address from the suppression list, see the Amazon SES Developer Guide: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/remove-from-suppressionlist.html \",\"emailAddress\":\"suppressed@example.edu\"}],\"timestamp\":\"2014-08-22T12:18:58.044Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"timestamp\":\"2014-08-22T12:18:57.000Z\",\"source\":\"notifications@instructure.com\",\"messageId\":\"example-id\",\"destination\":[\"suppressed@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:18:58.143Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
},
{
"Type" : "Notification",
"MessageId" : "example-id",
"TopicArn" : "example-arn",
"Message" : "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceSubType\":\"General\",\"bounceType\":\"Transient\",\"reportingMTA\":\"dsn; a14-1.smtp-out.amazonses.com\",\"bouncedRecipients\":[{\"emailAddress\":\"soft@example.edu\",\"status\":\"4.4.7\",\"action\":\"failed\",\"diagnosticCode\":\"smtp; 554 4.4.7 Message expired: unable to deliver in 840 minutes.<421 4.4.0 Unable to lookup DNS for ms-wasatch-edu.mail.protection.outlook.com>\"}],\"timestamp\":\"2014-08-22T12:18:58.226Z\",\"feedbackId\":\"example-id\"},\"mail\":{\"timestamp\":\"2014-08-21T20:51:56.000Z\",\"source\":\"notifications@instructure.com\",\"messageId\":\"example-id\",\"destination\":[\"soft@example.edu\"]}}",
"Timestamp" : "2014-08-22T12:18:58.393Z",
"SignatureVersion" : "1",
"Signature" : "example==",
"SigningCertURL" : "https://example.com/example.pem",
"UnsubscribeURL" : "https://example.com/unsubscribe"
}
]

View File

@ -308,6 +308,19 @@ describe NotificationMessageCreator do
NotificationPolicy.count.should == 2
end
it "should not send to bouncing channels" do
notification_set
@communication_channel.bounce_count = 1
@communication_channel.save!
messages = NotificationMessageCreator.new(@notification, @assignment, :to_list => @user).create_message
messages.select{|m| m.to == 'value for path'}.size.should == 1
@communication_channel.bounce_count = 100
@communication_channel.save!
messages = NotificationMessageCreator.new(@notification, @assignment, :to_list => @user).create_message
messages.select{|m| m.to == 'value for path'}.size.should == 0
end
it "should not use notification policies for unconfirmed communication channels" do
notification_set
cc = communication_channel_model(:workflow_state => 'unconfirmed', :path => "nope")

View File

@ -0,0 +1,58 @@
#
# Copyright (C) 2013 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
describe BounceNotificationProcessor do
before(:once) do
bounce_queue_log = File.read(File.dirname(__FILE__) + '/../fixtures/bounces.json')
@all_bounce_messages_json = JSON.parse(bounce_queue_log)
@soft_bounce_messages_json = @all_bounce_messages_json.select {|m| m['Message'].include?('Transient')}
@hard_bounce_messages_json = @all_bounce_messages_json.select {|m| m['Message'].include?('Permanent')}
end
def mock_message(json)
message = mock
message.stubs(:body).returns(json.to_json)
message
end
describe ".process" do
it "processes each notification in the queue" do
bnp = BounceNotificationProcessor.new(access_key: 'key', secret_access_key: 'secret')
queue = mock
queue.expects(:poll).multiple_yields(*@all_bounce_messages_json.map {|m| mock_message(m)})
bnp.stubs(:bounce_queue).returns(queue)
bnp.expects(:process_bounce_notification).times(@all_bounce_messages_json.size)
bnp.process
end
it "flags addresses with hard bounces" do
bnp = BounceNotificationProcessor.new(access_key: 'key', secret_access_key: 'secret')
queue = mock
queue.expects(:poll).multiple_yields(*@all_bounce_messages_json.map {|m| mock_message(m)})
bnp.stubs(:bounce_queue).returns(queue)
CommunicationChannel.expects(:bounce_for_path).with('hard@example.edu').times(5)
CommunicationChannel.expects(:bounce_for_path).with('suppressed@example.edu').times(1)
CommunicationChannel.expects(:bounce_for_path).with('soft@example.edu').times(0)
bnp.process
end
end
end

View File

@ -73,34 +73,6 @@ describe CommunicationChannel do
@cc.state.should eql(:active)
end
it "should reset the bounce count when re_activating" do
communication_channel_model
@cc.bounce_count = 1
@cc.confirm
@cc.bounce_count.should eql(1)
@cc.retire
@cc.re_activate
@cc.bounce_count.should eql(0)
end
it "should retire the communication channel if it's been bounced 5 times" do
communication_channel_model
@cc.bounce_count = 5
@cc.state.should eql(:unconfirmed)
@cc.save
@cc.state.should eql(:retired)
communication_channel_model
@cc.bounce_count = 4
@cc.save
@cc.state.should eql(:unconfirmed)
communication_channel_model
@cc.bounce_count = 6
@cc.save
@cc.state.should eql(:retired)
end
it "should set a confirmation code unless one has been set" do
CanvasSlug.expects(:generate).at_least(1).returns('abc123')
communication_channel_model
@ -274,6 +246,23 @@ describe CommunicationChannel do
cc1.has_merge_candidates?.should be_true
end
describe ".bounce_for_path" do
it "flags paths with too many bounces" do
@cc1 = communication_channel_model(path: 'not_as_bouncy@example.edu')
@cc2 = communication_channel_model(path: 'bouncy@example.edu')
%w{bouncy@example.edu Bouncy@example.edu bOuNcY@Example.edu bouncy@example.edu not_as_bouncy@example.edu bouncy@example.edu}.each{|path| CommunicationChannel.bounce_for_path(path)}
@cc1.reload
@cc1.bounce_count.should == 1
@cc1.bouncing?.should be_falsey
@cc2.reload
@cc2.bounce_count.should == 5
@cc2.bouncing?.should be_truthy
end
end
context "sharding" do
specs_require_sharding
@ -309,6 +298,35 @@ describe CommunicationChannel do
cc1.merge_candidates.should == []
@cc2.merge_candidates.should == []
end
describe ".bounce_for_path" do
it "flags paths with too many bounces" do
@cc1 = communication_channel_model(path: 'not_as_bouncy@example.edu')
@shard1.activate do
@cc2 = communication_channel_model(path: 'bouncy@example.edu')
end
pending if CommunicationChannel.associated_shards('bouncy@example.edu') == [Shard.default]
@shard2.activate do
@cc3 = communication_channel_model(path: 'BOUNCY@example.edu')
end
%w{bouncy@example.edu Bouncy@example.edu bOuNcY@Example.edu bouncy@example.edu not_as_bouncy@example.edu bouncy@example.edu}.each{|path| CommunicationChannel.bounce_for_path(path)}
@cc1.reload
@cc1.bounce_count.should == 1
@cc1.bouncing?.should be_falsey
@cc2.reload
@cc2.bounce_count.should == 5
@cc2.bouncing?.should be_truthy
@cc3.reload
@cc3.bounce_count.should == 5
@cc3.bouncing?.should be_truthy
end
end
end
end
end