update the redis gem to 3.0.1

This required building our own fork of the redis-store gem so that we
could update its dependency, and fix one small issue with redis connect
strings getting nil instead of the default value for the port number.

The redis 3.0.x gem now catches all Errno and Timeout errors and
re-raises them as subclasses of Redis::BaseConnectionError. It also now
handles EAGAIN internally, retrying when appropriate. So we've modified
our redis failure handling code to match.

test plan: verify the redis failure handling code still works (specs
pass). for instance, stop redis locally and see that canvas works in the
degraded state. make sure that redis still works for both caching and
non-caching code such as login attempts.

Change-Id: I9e8d3929afa06c522656d30f71efc0427e4ef7cc
Reviewed-on: https://gerrit.instructure.com/11521
Tested-by: Jenkins <jenkins@instructure.com>
Reviewed-by: Cody Cutrer <cody@instructure.com>
This commit is contained in:
Brian Palmer 2012-06-12 10:17:36 -06:00
parent f8f6627602
commit e66fa507cf
4 changed files with 5 additions and 36 deletions

View File

@ -114,7 +114,8 @@ group :i18n_tools do
end
group :redis do
gem 'redis-store', '1.0.0.rc1'
gem 'instructure-redis-store', '1.0.0.1.instructure1', :require => 'redis-store'
gem 'redis', '3.0.1'
end
group :embedly do

View File

@ -38,27 +38,13 @@ module Canvas::Redis
def self.handle_redis_failure(failure_retval)
return failure_retval if redis_failure?
yield
rescue Errno::ECONNREFUSED, Timeout::Error, Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL => e
rescue Redis::BaseConnectionError => e
ErrorReport.log_exception(:redis, e)
Rails.logger.error "Failure handling redis command: #{e.inspect}"
@last_redis_failure = Time.now
failure_retval
rescue Errno::EAGAIN => e
ErrorReport.log_exception(:redis, e)
Rails.logger.error "Redis EAGAIN failure, trying again: #{e.inspect}"
# typically this means that redis closed the connection as idle,
# and trying again will succeed
begin
yield
rescue Errno::EAGAIN => e
ErrorReport.log_exception(:redis, e)
Rails.logger.error "Failure handling redis command: #{e.inspect}"
failure_retval
end
end
# while we wait for this pull request
# https://github.com/jodosha/redis-store/pull/83
def self.patch
return if @redis_patched
Redis::Client.class_eval do

View File

@ -46,28 +46,10 @@ describe "Canvas::Redis" do
end
end
it "should try again on redis EAGAIN" do
Canvas::Redis.patch
Redis::Client.any_instance.expects(:ensure_connected).twice.raises(Errno::EAGAIN).then.returns(Marshal.dump("success!"))
enable_cache(ActiveSupport::Cache::RedisStore.new(['redis://localhost:1234'])) do
Rails.cache.read('blah').should == "success!"
end
Canvas::Redis.reset_redis_failure
end
it "should give up after two EAGAINs" do
Canvas::Redis.patch
Redis::Client.any_instance.expects(:ensure_connected).twice.raises(Errno::EAGAIN)
enable_cache(ActiveSupport::Cache::RedisStore.new(['redis://localhost:1234'])) do
Rails.cache.read('blah').should == nil
end
Canvas::Redis.reset_redis_failure
end
describe "redis failure" do
before do
Canvas::Redis.patch
Redis::Client.any_instance.expects(:ensure_connected).raises(Timeout::Error).once
Redis::Client.any_instance.expects(:ensure_connected).raises(Redis::TimeoutError).once
end
after do

View File

@ -58,7 +58,7 @@ describe PageView do
it "should store directly to the db if redis is down" do
Canvas::Redis.patch
Redis::Client.any_instance.expects(:ensure_connected).raises(Timeout::Error)
Redis::Client.any_instance.expects(:ensure_connected).raises(Redis::TimeoutError)
@page_view.store.should be_true
PageView.count.should == 1
PageView.first.attributes.except('created_at', 'updated_at').should == @page_view.attributes.except('created_at', 'updated_at')