update the redis gem to 3.0.1
This required building our own fork of the redis-store gem so that we could update its dependency, and fix one small issue with redis connect strings getting nil instead of the default value for the port number. The redis 3.0.x gem now catches all Errno and Timeout errors and re-raises them as subclasses of Redis::BaseConnectionError. It also now handles EAGAIN internally, retrying when appropriate. So we've modified our redis failure handling code to match. test plan: verify the redis failure handling code still works (specs pass). for instance, stop redis locally and see that canvas works in the degraded state. make sure that redis still works for both caching and non-caching code such as login attempts. Change-Id: I9e8d3929afa06c522656d30f71efc0427e4ef7cc Reviewed-on: https://gerrit.instructure.com/11521 Tested-by: Jenkins <jenkins@instructure.com> Reviewed-by: Cody Cutrer <cody@instructure.com>
This commit is contained in:
parent
f8f6627602
commit
e66fa507cf
3
Gemfile
3
Gemfile
|
@ -114,7 +114,8 @@ group :i18n_tools do
|
|||
end
|
||||
|
||||
group :redis do
|
||||
gem 'redis-store', '1.0.0.rc1'
|
||||
gem 'instructure-redis-store', '1.0.0.1.instructure1', :require => 'redis-store'
|
||||
gem 'redis', '3.0.1'
|
||||
end
|
||||
|
||||
group :embedly do
|
||||
|
|
|
@ -38,27 +38,13 @@ module Canvas::Redis
|
|||
def self.handle_redis_failure(failure_retval)
|
||||
return failure_retval if redis_failure?
|
||||
yield
|
||||
rescue Errno::ECONNREFUSED, Timeout::Error, Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL => e
|
||||
rescue Redis::BaseConnectionError => e
|
||||
ErrorReport.log_exception(:redis, e)
|
||||
Rails.logger.error "Failure handling redis command: #{e.inspect}"
|
||||
@last_redis_failure = Time.now
|
||||
failure_retval
|
||||
rescue Errno::EAGAIN => e
|
||||
ErrorReport.log_exception(:redis, e)
|
||||
Rails.logger.error "Redis EAGAIN failure, trying again: #{e.inspect}"
|
||||
# typically this means that redis closed the connection as idle,
|
||||
# and trying again will succeed
|
||||
begin
|
||||
yield
|
||||
rescue Errno::EAGAIN => e
|
||||
ErrorReport.log_exception(:redis, e)
|
||||
Rails.logger.error "Failure handling redis command: #{e.inspect}"
|
||||
failure_retval
|
||||
end
|
||||
end
|
||||
|
||||
# while we wait for this pull request
|
||||
# https://github.com/jodosha/redis-store/pull/83
|
||||
def self.patch
|
||||
return if @redis_patched
|
||||
Redis::Client.class_eval do
|
||||
|
|
|
@ -46,28 +46,10 @@ describe "Canvas::Redis" do
|
|||
end
|
||||
end
|
||||
|
||||
it "should try again on redis EAGAIN" do
|
||||
Canvas::Redis.patch
|
||||
Redis::Client.any_instance.expects(:ensure_connected).twice.raises(Errno::EAGAIN).then.returns(Marshal.dump("success!"))
|
||||
enable_cache(ActiveSupport::Cache::RedisStore.new(['redis://localhost:1234'])) do
|
||||
Rails.cache.read('blah').should == "success!"
|
||||
end
|
||||
Canvas::Redis.reset_redis_failure
|
||||
end
|
||||
|
||||
it "should give up after two EAGAINs" do
|
||||
Canvas::Redis.patch
|
||||
Redis::Client.any_instance.expects(:ensure_connected).twice.raises(Errno::EAGAIN)
|
||||
enable_cache(ActiveSupport::Cache::RedisStore.new(['redis://localhost:1234'])) do
|
||||
Rails.cache.read('blah').should == nil
|
||||
end
|
||||
Canvas::Redis.reset_redis_failure
|
||||
end
|
||||
|
||||
describe "redis failure" do
|
||||
before do
|
||||
Canvas::Redis.patch
|
||||
Redis::Client.any_instance.expects(:ensure_connected).raises(Timeout::Error).once
|
||||
Redis::Client.any_instance.expects(:ensure_connected).raises(Redis::TimeoutError).once
|
||||
end
|
||||
|
||||
after do
|
||||
|
|
|
@ -58,7 +58,7 @@ describe PageView do
|
|||
|
||||
it "should store directly to the db if redis is down" do
|
||||
Canvas::Redis.patch
|
||||
Redis::Client.any_instance.expects(:ensure_connected).raises(Timeout::Error)
|
||||
Redis::Client.any_instance.expects(:ensure_connected).raises(Redis::TimeoutError)
|
||||
@page_view.store.should be_true
|
||||
PageView.count.should == 1
|
||||
PageView.first.attributes.except('created_at', 'updated_at').should == @page_view.attributes.except('created_at', 'updated_at')
|
||||
|
|
Loading…
Reference in New Issue