From f647a1289e6625fb9aaa751e98779078d397d239 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 31 May 2023 16:39:45 -0700 Subject: [PATCH] Split GLOBAL_TAG_THROTTLING_FOLDING_TIME into several knobs --- design/global-tag-throttling.md | 2 +- fdbclient/ServerKnobs.cpp | 12 +++++------- fdbclient/include/fdbclient/ServerKnobs.h | 7 +++++-- fdbserver/GlobalTagThrottler.actor.cpp | 15 ++++++++------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/design/global-tag-throttling.md b/design/global-tag-throttling.md index 9166246212..4fc90d9cec 100644 --- a/design/global-tag-throttling.md +++ b/design/global-tag-throttling.md @@ -80,7 +80,7 @@ The ratekeeper must also track the rate of transactions performed with each tag. ### Average Cost Calculation Quotas are expressed in terms of cost, but because throttling is enforced at the beginning of transactions, budgets need to be calculated in terms of transactions per second. To make this conversion, it is necessary to track the average cost of transactions (per-tag, and per-tag on a particular storage server). -Both cost and transaction counters are smoothed using the `Smoother` class to provide stability over time. The "smoothing interval" can be modified through `SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME`. +Both cost and transaction counters are exponentially smoothed over time, with knob-configurable smoothing intervals. ### Reserved Rate Calculation The global tag throttler periodically reads reserved quotas from the system keyspace. Using these reserved quotas and the average cost of transactions with the given tag, a reserved TPS rate is computed. Read and write rates are aggregated as follows: diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 5ac9196f6d..fcaac756f8 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -833,19 +833,17 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( GLOBAL_TAG_THROTTLING, true ); if(isSimulated) GLOBAL_TAG_THROTTLING = deterministicRandom()->coinflip(); init( ENFORCE_TAG_THROTTLING_ON_PROXIES, GLOBAL_TAG_THROTTLING ); init( GLOBAL_TAG_THROTTLING_MIN_RATE, 1.0 ); - // 10 seconds was chosen as a default value to ensure that - // the global tag throttler does not react too drastically to - // changes in workload. To make the global tag throttler more reactive, - // lower this knob. To make global tag throttler more smooth, raise this knob. - // Setting this knob lower than TAG_MEASUREMENT_INTERVAL can cause erratic - // behaviour and is not recommended. - init( GLOBAL_TAG_THROTTLING_FOLDING_TIME, 10.0 ); init( GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED, 10 ); init( GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER, 240.0 ); init( GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL, 60.0 ); init( GLOBAL_TAG_THROTTLING_TRACE_INTERVAL, 5.0 ); init( GLOBAL_TAG_THROTTLING_REPORT_ONLY, false ); + init( GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME, 10.0 ); + init( GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME, 2.0 ); + init( GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME, 10.0 ); + init( GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME, 10.0 ); + //Storage Metrics init( STORAGE_METRICS_AVERAGE_INTERVAL, 120.0 ); init( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS, 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ); // milliHz! diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 5eb4b04c5d..f02abfa505 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -762,8 +762,6 @@ public: // To protect against this, we do not compute the average cost when the // measured tps drops below this threshold double GLOBAL_TAG_THROTTLING_MIN_RATE; - // Used by global tag throttling counters - double GLOBAL_TAG_THROTTLING_FOLDING_TIME; // Maximum number of tags tracked by global tag throttler. Additional tags will be ignored // until some existing tags expire int64_t GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED; @@ -779,6 +777,11 @@ public: // enforcement. bool GLOBAL_TAG_THROTTLING_REPORT_ONLY; + double GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME; + double GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME; + double GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME; + double GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME; + double MAX_TRANSACTIONS_PER_BYTE; int64_t MIN_AVAILABLE_SPACE; diff --git a/fdbserver/GlobalTagThrottler.actor.cpp b/fdbserver/GlobalTagThrottler.actor.cpp index 3d2cf90f9e..cf6a8a2ed5 100644 --- a/fdbserver/GlobalTagThrottler.actor.cpp +++ b/fdbserver/GlobalTagThrottler.actor.cpp @@ -100,8 +100,8 @@ class GlobalTagThrottlerImpl { public: ThroughputCounters() - : readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME), - writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {} + : readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME), + writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {} void updateCost(double newCost, OpType opType) { if (opType == OpType::READ) { @@ -125,10 +125,11 @@ class GlobalTagThrottlerImpl { public: explicit PerTagStatistics() - : transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME, - SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME), - perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME), - targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME), transactionsLastAdded(now()), lastLogged(0) {} + : transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME, + SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME), + perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME), + targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME), transactionsLastAdded(now()), + lastLogged(0) {} Optional getQuota() const { return quota; } @@ -667,7 +668,7 @@ class MockStorageServer { Smoother smoother; public: - Cost() : smoother(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {} + Cost() : smoother(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {} Cost& operator+=(double delta) { smoother.addDelta(delta); return *this;