Add panic knob to shut down cache from outside txn options

This commit is contained in:
Jon Fu 2021-11-25 13:22:28 -05:00
parent beae9ccfa1
commit bba7ed8ac0
3 changed files with 9 additions and 3 deletions

View File

@ -120,6 +120,7 @@ void ClientKnobs::initialize(Randomize randomize) {
init( MAX_VERSION_CACHE_LAG, 0.1 );
init( MAX_PROXY_CONTACT_LAG, 1.0 );
init( DEBUG_USE_GRV_CACHE_CHANCE, -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0.
init( FORCE_GRV_CACHE_OFF, false );
init( GRV_CACHE_RK_COOLDOWN, 60.0 );
init( GRV_THROTTLING_THRESHOLD, 100e-9 );
init( GRV_SUSTAINED_THROTTLING_THRESHOLD, 1.0 );

View File

@ -123,7 +123,7 @@ public:
double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
// should these be server knobs?
double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
double GRV_THROTTLING_THRESHOLD; // Number of seconds spent in the request queue which is considered throttling

View File

@ -31,6 +31,7 @@
#include "contrib/fmt-8.0.1/include/fmt/format.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/Knobs.h"
#include "fdbrpc/FailureMonitor.h"
#include "fdbrpc/MultiInterface.h"
@ -6033,8 +6034,9 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
Future<Version> Transaction::getReadVersion(uint32_t flags) {
if (!readVersion.isValid()) {
if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
(deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache)) {
if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
(deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
rkThrottlingCooledDown(getDatabase().getPtr())) {
TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", format("%.6f", cx->lastTimedGrv.get()));
// Upon our first request to use cached RVs, start the background updater
if (!cx->grvUpdateHandler.isValid()) {
@ -6043,6 +6045,9 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
return readVersion;
}
if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && cx->grvUpdateHandler.isValid()) {
cx->grvUpdateHandler.cancel();
}
++cx->transactionReadVersions;
flags |= options.getReadVersionFlags;
switch (options.priority) {