wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split GLOBAL_TAG_THROTTLING_FOLDING_TIME into several knobs

This commit is contained in:
sfc-gh-tclinkenbeard 2023-05-31 16:39:45 -07:00
parent 11b08c4060
commit f647a1289e
4 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
design/global-tag-throttling.md
View File

@ -80,7 +80,7 @@ The ratekeeper must also track the rate of transactions performed with each tag.
### Average Cost Calculation
Quotas are expressed in terms of cost, but because throttling is enforced at the beginning of transactions, budgets need to be calculated in terms of transactions per second. To make this conversion, it is necessary to track the average cost of transactions (per-tag, and per-tag on a particular storage server).
Both cost and transaction counters are smoothed using the `Smoother` class to provide stability over time. The "smoothing interval" can be modified through `SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME`.
Both cost and transaction counters are exponentially smoothed over time, with knob-configurable smoothing intervals.
### Reserved Rate Calculation
The global tag throttler periodically reads reserved quotas from the system keyspace. Using these reserved quotas and the average cost of transactions with the given tag, a reserved TPS rate is computed. Read and write rates are aggregated as follows:

12
fdbclient/ServerKnobs.cpp
View File

@ -833,19 +833,17 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( GLOBAL_TAG_THROTTLING, true ); if(isSimulated) GLOBAL_TAG_THROTTLING = deterministicRandom()->coinflip();
init( ENFORCE_TAG_THROTTLING_ON_PROXIES, GLOBAL_TAG_THROTTLING );
init( GLOBAL_TAG_THROTTLING_MIN_RATE, 1.0 );
// 10 seconds was chosen as a default value to ensure that
// the global tag throttler does not react too drastically to
// changes in workload. To make the global tag throttler more reactive,
// lower this knob. To make global tag throttler more smooth, raise this knob.
// Setting this knob lower than TAG_MEASUREMENT_INTERVAL can cause erratic
// behaviour and is not recommended.
init( GLOBAL_TAG_THROTTLING_FOLDING_TIME, 10.0 );
init( GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED, 10 );
init( GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER, 240.0 );
init( GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL, 60.0 );
init( GLOBAL_TAG_THROTTLING_TRACE_INTERVAL, 5.0 );
init( GLOBAL_TAG_THROTTLING_REPORT_ONLY, false );
init( GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME, 10.0 );
init( GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME, 2.0 );
init( GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME, 10.0 );
init( GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME, 10.0 );
//Storage Metrics
init( STORAGE_METRICS_AVERAGE_INTERVAL, 120.0 );
init( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS, 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ); // milliHz!

7
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -762,8 +762,6 @@ public:
// To protect against this, we do not compute the average cost when the
// measured tps drops below this threshold
double GLOBAL_TAG_THROTTLING_MIN_RATE;
// Used by global tag throttling counters
double GLOBAL_TAG_THROTTLING_FOLDING_TIME;
// Maximum number of tags tracked by global tag throttler. Additional tags will be ignored
// until some existing tags expire
int64_t GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED;
@ -779,6 +777,11 @@ public:
// enforcement.
bool GLOBAL_TAG_THROTTLING_REPORT_ONLY;
double GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME;
double GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME;
double GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME;
double GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME;
double MAX_TRANSACTIONS_PER_BYTE;
int64_t MIN_AVAILABLE_SPACE;

15
fdbserver/GlobalTagThrottler.actor.cpp
View File

@ -100,8 +100,8 @@ class GlobalTagThrottlerImpl {
public:
ThroughputCounters()
: readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {}
: readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME),
writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {}
void updateCost(double newCost, OpType opType) {
if (opType == OpType::READ) {
@ -125,10 +125,11 @@ class GlobalTagThrottlerImpl {
public:
explicit PerTagStatistics()
: transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME,
SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME), transactionsLastAdded(now()), lastLogged(0) {}
: transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME,
SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME),
perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME),
targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME), transactionsLastAdded(now()),
lastLogged(0) {}
Optional<ThrottleApi::TagQuotaValue> getQuota() const { return quota; }
@ -667,7 +668,7 @@ class MockStorageServer {
Smoother smoother;
public:
Cost() : smoother(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {}
Cost() : smoother(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {}
Cost& operator+=(double delta) {
smoother.addDelta(delta);
return *this;