From e2cdffce3d4086e33db172cf9722e84f86a80e84 Mon Sep 17 00:00:00 2001 From: Cliff Pruitt Date: Mon, 9 Dec 2019 14:48:14 -0500 Subject: [PATCH 1/2] Add config option for ActiveJob::Base.default_retry_jitter --- activejob/lib/active_job/exceptions.rb | 10 ++++++-- activejob/test/cases/exceptions_test.rb | 25 +++++++++++++++++++ .../lib/rails/application/configuration.rb | 4 +++ .../test/application/configuration_test.rb | 14 +++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/activejob/lib/active_job/exceptions.rb b/activejob/lib/active_job/exceptions.rb index f60240fe54e..c7d03303590 100644 --- a/activejob/lib/active_job/exceptions.rb +++ b/activejob/lib/active_job/exceptions.rb @@ -7,6 +7,11 @@ module ActiveJob module Exceptions extend ActiveSupport::Concern + included do + class_attribute :default_retry_jitter, instance_accessor: false, instance_predicate: false + self.default_retry_jitter = 0.15 + end + module ClassMethods # Catch the exception and reschedule job for re-execution after so many seconds, for a specific number of attempts. # If the exception keeps getting raised beyond the specified number of attempts, the exception is allowed to @@ -49,7 +54,7 @@ module ActiveJob # # Might raise Net::OpenTimeout or Timeout::Error when the remote service is down # end # end - def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: 0.15) + def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: nil) rescue_from(*exceptions) do |error| executions = executions_for(exceptions) if executions < attempts @@ -122,7 +127,8 @@ module ActiveJob end private - def determine_delay(seconds_or_duration_or_algorithm:, executions:, jitter:) + def determine_delay(seconds_or_duration_or_algorithm:, executions:, jitter: nil) + jitter ||= self.class.default_retry_jitter case seconds_or_duration_or_algorithm when :exponentially_longer ((executions**4) + (Kernel.rand((executions**4) * jitter))) + 2 diff --git a/activejob/test/cases/exceptions_test.rb b/activejob/test/cases/exceptions_test.rb index 277016c9d26..96f3babaddd 100644 --- a/activejob/test/cases/exceptions_test.rb +++ b/activejob/test/cases/exceptions_test.rb @@ -129,6 +129,31 @@ class ExceptionsTest < ActiveSupport::TestCase end end + test "retry jitter uses value from ActiveJob::Base.default_retry_jitter by default" do + old_jitter = ActiveJob::Base.default_retry_jitter + ActiveJob::Base.default_retry_jitter = 4.0 + + travel_to Time.now + + Kernel.stub(:rand, ->(arg) { arg }) do + RetryJob.perform_later "ExponentialWaitTenAttemptsError", 5, :log_scheduled_at + + assert_equal [ + "Raised ExponentialWaitTenAttemptsError for the 1st time", + "Next execution scheduled at #{(Time.now + 7.seconds).to_f}", + "Raised ExponentialWaitTenAttemptsError for the 2nd time", + "Next execution scheduled at #{(Time.now + 82.seconds).to_f}", + "Raised ExponentialWaitTenAttemptsError for the 3rd time", + "Next execution scheduled at #{(Time.now + 407.seconds).to_f}", + "Raised ExponentialWaitTenAttemptsError for the 4th time", + "Next execution scheduled at #{(Time.now + 1282.seconds).to_f}", + "Successfully completed job" + ], JobBuffer.values + end + ensure + ActiveJob::Base.default_retry_jitter = old_jitter + end + test "custom wait retrying job" do travel_to Time.now diff --git a/railties/lib/rails/application/configuration.rb b/railties/lib/rails/application/configuration.rb index d57a86006dd..317bdea3880 100644 --- a/railties/lib/rails/application/configuration.rb +++ b/railties/lib/rails/application/configuration.rb @@ -156,6 +156,10 @@ module Rails when "6.1" load_defaults "6.0" + if respond_to?(:active_job) + active_job.default_retry_jitter = 0.15 + end + if respond_to?(:active_record) active_record.has_many_inversing = true end diff --git a/railties/test/application/configuration_test.rb b/railties/test/application/configuration_test.rb index 7084185a335..aee9fb7ae46 100644 --- a/railties/test/application/configuration_test.rb +++ b/railties/test/application/configuration_test.rb @@ -2262,6 +2262,20 @@ module ApplicationTests end end + test "ActiveJob::Base.default_retry_jitter is 0.15 by default" do + app "development" + + assert_equal 0.15, ActiveJob::Base.default_retry_jitter + end + + test "ActiveJob::Base.default_retry_jitter can be set by config" do + app "development" + + Rails.application.config.active_job.default_retry_jitter = 0.22 + + assert_equal 0.22, ActiveJob::Base.default_retry_jitter + end + test "ActiveJob::Base.return_false_on_aborted_enqueue is true by default" do app "development" From 64649cec8b3eb2d78cb299977ad62c3ad633d4e7 Mon Sep 17 00:00:00 2001 From: Cliff Pruitt Date: Tue, 10 Dec 2019 10:00:24 -0500 Subject: [PATCH 2/2] Add explaination of default_retry_jitter to configuring guide --- guides/source/configuring.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/guides/source/configuring.md b/guides/source/configuring.md index c262a49c304..e4656cae2c2 100644 --- a/guides/source/configuring.md +++ b/guides/source/configuring.md @@ -807,6 +807,8 @@ There are a few configuration options available in Active Support: * `config.active_job.log_arguments` controls if the arguments of a job are logged. Defaults to `true`. +* `config.active_job.default_retry_jitter` controls the amount of "jitter" (random variation) applied to the delay time calculated when retrying failed jobs. Defaults to `0.15`. + ### Configuring Action Cable * `config.action_cable.url` accepts a string for the URL for where