From 1ce2457f7eafe398aaf53cadba7dafbc3aa9f4f0 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 17 Aug 2021 15:12:02 -0400
Subject: [PATCH 001/210] initial commit

---
 fdbclient/DatabaseContext.h          |  6 ++++
 fdbclient/Knobs.cpp                  |  2 ++
 fdbclient/Knobs.h                    |  1 +
 fdbclient/NativeAPI.actor.cpp        | 45 ++++++++++++++++++++++++++++
 fdbclient/NativeAPI.actor.h          |  1 +
 fdbclient/ReadYourWrites.actor.cpp   |  3 ++
 fdbclient/ReadYourWrites.h           |  1 +
 fdbclient/vexillographer/fdb.options |  2 ++
 8 files changed, 61 insertions(+)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 52c1945f8e..d06e0a2cda 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -276,6 +276,7 @@ public:
 	Future<Void> monitorTssInfoChange;
 	Future<Void> tssMismatchHandler;
 	PromiseStream<UID> tssMismatchStream;
+	Future<Void> grvUpdateHandler;
 	Reference<CommitProxyInfo> commitProxies;
 	Reference<GrvProxyInfo> grvProxies;
 	bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time.
@@ -328,6 +329,11 @@ public:
 	// map from tssid -> metrics for that tss pair
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 
+	// Database-level read version cache storing the most recent successful GRV
+	Future<Version> cachedReadVersion;
+	double lastGrvUpdateTime;
+	void updateCachedRV(Future<Version> v);
+
 	UID dbId;
 	bool internal; // Only contexts created through the C client and fdbcli are non-internal
 
diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp
index 462c6f733c..9b058392d8 100644
--- a/fdbclient/Knobs.cpp
+++ b/fdbclient/Knobs.cpp
@@ -114,6 +114,8 @@ void ClientKnobs::initialize(bool randomize) {
 	init( CORE_VERSIONSPERSECOND,		           1e6 );
 	init( LOG_RANGE_BLOCK_SIZE,                    1e6 ); //Dependent on CORE_VERSIONSPERSECOND
 	init( MUTATION_BLOCK_SIZE,	            	  10000 );
+	init( MAX_VERSION_CACHE_LAG,                   10.0 ); // If subtracting time from now(), what unit is being measured?
+
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h
index f01624d7fe..ea1d3e41bc 100644
--- a/fdbclient/Knobs.h
+++ b/fdbclient/Knobs.h
@@ -110,6 +110,7 @@ public:
 	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
 	int LOG_RANGE_BLOCK_SIZE;
 	int MUTATION_BLOCK_SIZE;
+	double MAX_VERSION_CACHE_LAG;
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 3f6ed7530b..3d11310fc9 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -53,6 +53,7 @@
 #include "fdbrpc/LoadBalance.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
+#include "fdbserver/Knobs.h"
 #include "flow/Arena.h"
 #include "flow/ActorCollection.h"
 #include "flow/DeterministicRandom.h"
@@ -168,6 +169,11 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
+void DatabaseContext::updateCachedRV(Future<Version> v) {
+	lastGrvUpdateTime = now();
+	cachedReadVersion = v;
+}
+
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
                                                              StorageServerInterface const& ssi,
                                                              LocalityData const& locality) {
@@ -879,6 +885,28 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 	}
 }
 
+ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
+	cx->lastGrvUpdateTime = 0.0;
+	state Transaction tr;
+	loop {
+		wait(refreshTransaction(cx, &tr));
+		auto curTime = now();
+		if (curTime - cx->lastGrvUpdateTime > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+			try {
+				// Is this what the method should be? Maybe skip transaction
+				// and go straight to proxy like getRawVersion()
+				Version v = wait(tr.getReadVersion());
+				cx->updateCachedRV(v);
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled) {
+					throw;
+				}
+				wait(tr.onError(e));
+			}
+		}
+	}
+}
+
 ACTOR static Future<HealthMetrics> getHealthMetricsActor(DatabaseContext* cx, bool detailed) {
 	if (now() - cx->healthMetricsLastUpdated < CLIENT_KNOBS->AGGREGATE_HEALTH_METRICS_MAX_STALENESS) {
 		if (detailed) {
@@ -1127,6 +1155,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 
 	monitorProxiesInfoChange = monitorProxiesChange(clientInfo, &proxiesChangeTrigger);
 	tssMismatchHandler = handleTssMismatches(this);
+	grvUpdateHandler = backgroundGrvUpdater(this);
 	clientStatusUpdater.actor = clientStatusUpdateActor(this);
 	cacheListMonitor = monitorCacheList(this);
 
@@ -1370,6 +1399,7 @@ DatabaseContext::~DatabaseContext() {
 	monitorProxiesInfoChange.cancel();
 	monitorTssInfoChange.cancel();
 	tssMismatchHandler.cancel();
+	grvUpdateHandler.cancel();
 	for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it))
 		it->second->notifyContextDestroyed();
 	ASSERT_ABORT(server_interf.empty());
@@ -3584,6 +3614,7 @@ void Transaction::setVersion(Version v) {
 	if (v <= 0)
 		throw version_invalid();
 	readVersion = v;
+	cx->updateCachedRV(v);
 }
 
 Future<Optional<Value>> Transaction::get(const Key& key, bool snapshot) {
@@ -4059,6 +4090,7 @@ void TransactionOptions::clear() {
 	readTags = TagSet{};
 	priority = TransactionPriority::DEFAULT;
 	expensiveClearCostEstimation = false;
+	useGrvCache = false;
 }
 
 TransactionOptions::TransactionOptions() {
@@ -4596,6 +4628,8 @@ Future<Void> Transaction::commitMutations() {
 		Future<Void> commitResult =
 		    tryCommit(cx, trLogInfo, tr, readVersion, info, &this->committedVersion, this, options);
 
+		// Record committed version if successful (failed commit would go to catch block instead(?))
+		cx->updateCachedRV(this->committedVersion);
 		if (isCheckingWrites) {
 			Promise<Void> committed;
 			checkWrites(cx, commitResult, committed, tr, this);
@@ -4823,6 +4857,12 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		options.expensiveClearCostEstimation = true;
 		break;
 
+	case FDBTransactionOptions::USE_GRV_CACHE:
+		// Needs to be a no-op if ratekeeper is throttling
+		validateOptionValue(value, false);
+		options.useGrvCache = true;
+		break;
+
 	default:
 		break;
 	}
@@ -5031,6 +5071,10 @@ ACTOR Future<Version> extractReadVersion(Location location,
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
+		if (options.useGrvCache && cx->cachedReadVersion.isValid() && cx->cachedReadVersion.isReady() &&
+		    !cx->cachedReadVersion.isError()) {
+			return cx->cachedReadVersion;
+		}
 		++cx->transactionReadVersions;
 		flags |= options.getReadVersionFlags;
 		switch (options.priority) {
@@ -5105,6 +5149,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		                                 startTime,
 		                                 metadataVersion,
 		                                 options.tags);
+		cx->updateCachedRV(readVersion);
 	}
 	return readVersion;
 }
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index 2955a792dd..849d937c62 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -148,6 +148,7 @@ struct TransactionOptions {
 	bool includePort : 1;
 	bool reportConflictingKeys : 1;
 	bool expensiveClearCostEstimation : 1;
+	bool useGrvCache : 1;
 
 	TransactionPriority priority;
 
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index 6c7b77ff65..e2ededfd5f 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2239,6 +2239,9 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 		validateOptionValue(value, false);
 		options.bypassUnreadable = true;
 		break;
+	case FDBTransactionOptions::USE_GRV_CACHE:
+		validateOptionValue(value, false);
+		options.useGrvCache = true;
 	default:
 		break;
 	}
diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h
index 977dab1f24..41db431de4 100644
--- a/fdbclient/ReadYourWrites.h
+++ b/fdbclient/ReadYourWrites.h
@@ -43,6 +43,7 @@ struct ReadYourWritesTransactionOptions {
 	int maxRetries;
 	int snapshotRywEnabled;
 	bool bypassUnreadable : 1;
+	bool useGrvCache : 1;
 
 	ReadYourWritesTransactionOptions() {}
 	explicit ReadYourWritesTransactionOptions(Transaction const& tr);
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 2d3a5b57ce..faa05066ff 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -289,6 +289,8 @@ description is not currently required but encouraged.
                 description="Asks storage servers for how many bytes a clear key range contains. Otherwise uses the location cache to roughly estimate this." />
     <Option name="bypass_unreadable" code="1100"
                 description="Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp." />            
+    <Option name="use_grv_cache" code="1101"
+            description="Allows this transaction to use cached GRV from the database context." />
   </Scope>
 
   <!-- The enumeration values matter - do not change them without

From cfbe5eca9cc7a37c167aa49915acdaaa061d8df1 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 10 Sep 2021 15:06:48 -0400
Subject: [PATCH 002/210] Use notifieddouble to track GRV request time and
 include grvdelay in timing calculations

---
 fdbclient/DatabaseContext.h   |  9 +++----
 fdbclient/Knobs.cpp           |  2 +-
 fdbclient/NativeAPI.actor.cpp | 45 ++++++++++++++++++++++-------------
 3 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index d06e0a2cda..62f0b5961d 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -32,6 +32,7 @@
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/Notified.h"
 #include "fdbrpc/QueueModel.h"
 #include "fdbrpc/MultiInterface.h"
 #include "flow/TDMetric.actor.h"
@@ -329,10 +330,10 @@ public:
 	// map from tssid -> metrics for that tss pair
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 
-	// Database-level read version cache storing the most recent successful GRV
-	Future<Version> cachedReadVersion;
-	double lastGrvUpdateTime;
-	void updateCachedRV(Future<Version> v);
+	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
+	NotifiedDouble lastTimedGrv;
+	Version cachedRv;
+	void updateCachedRV(double t, Version v);
 
 	UID dbId;
 	bool internal; // Only contexts created through the C client and fdbcli are non-internal
diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp
index 9b058392d8..c4cdb5aab8 100644
--- a/fdbclient/Knobs.cpp
+++ b/fdbclient/Knobs.cpp
@@ -114,7 +114,7 @@ void ClientKnobs::initialize(bool randomize) {
 	init( CORE_VERSIONSPERSECOND,		           1e6 );
 	init( LOG_RANGE_BLOCK_SIZE,                    1e6 ); //Dependent on CORE_VERSIONSPERSECOND
 	init( MUTATION_BLOCK_SIZE,	            	  10000 );
-	init( MAX_VERSION_CACHE_LAG,                   10.0 ); // If subtracting time from now(), what unit is being measured?
+	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 
 
 	// TaskBucket
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 3d11310fc9..c86a9c7f77 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -169,9 +169,11 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-void DatabaseContext::updateCachedRV(Future<Version> v) {
-	lastGrvUpdateTime = now();
-	cachedReadVersion = v;
+void DatabaseContext::updateCachedRV(double t, Version v) {
+	if (t >= lastTimedGrv.get()) {
+		lastTimedGrv = t;
+		cachedRv = v;
+	}
 }
 
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
@@ -886,23 +888,27 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 }
 
 ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
-	cx->lastGrvUpdateTime = 0.0;
 	state Transaction tr;
+	state double grvDelay = 0.001;
+	cx->lastTimedGrv = 0.0;
+	cx->cachedRv = Version(0);
 	loop {
 		wait(refreshTransaction(cx, &tr));
-		auto curTime = now();
-		if (curTime - cx->lastGrvUpdateTime > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+		state double curTime = now();
+		state double lastTime = cx->lastTimedGrv.get();
+		if (curTime - lastTime > (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
 			try {
-				// Is this what the method should be? Maybe skip transaction
-				// and go straight to proxy like getRawVersion()
 				Version v = wait(tr.getReadVersion());
-				cx->updateCachedRV(v);
+				// Take the average of what we expect and what we got as the delay
+				grvDelay = (grvDelay + (now() - curTime)) / 2.0;
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled) {
 					throw;
 				}
 				wait(tr.onError(e));
 			}
+		} else {
+			wait(delay((CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)));
 		}
 	}
 }
@@ -3614,7 +3620,6 @@ void Transaction::setVersion(Version v) {
 	if (v <= 0)
 		throw version_invalid();
 	readVersion = v;
-	cx->updateCachedRV(v);
 }
 
 Future<Optional<Value>> Transaction::get(const Key& key, bool snapshot) {
@@ -4429,7 +4434,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			                         TaskPriority::DefaultPromiseEndpoint,
 			                         true);
 		}
-
+		state double grvTime = now();
 		choose {
 			when(wait(cx->onProxiesChanged())) {
 				reply.cancel();
@@ -4437,6 +4442,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
+				cx->updateCachedRV(grvTime, v);
 				if (v != invalidVersion) {
 					if (CLIENT_BUGGIFY) {
 						throw commit_unknown_result();
@@ -4628,8 +4634,6 @@ Future<Void> Transaction::commitMutations() {
 		Future<Void> commitResult =
 		    tryCommit(cx, trLogInfo, tr, readVersion, info, &this->committedVersion, this, options);
 
-		// Record committed version if successful (failed commit would go to catch block instead(?))
-		cx->updateCachedRV(this->committedVersion);
 		if (isCheckingWrites) {
 			Promise<Void> committed;
 			checkWrites(cx, commitResult, committed, tr, this);
@@ -5012,6 +5016,7 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	state Span span(spanContext, location, { parent });
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
+	cx->updateCachedRV(startTime, rep.version);
 	cx->GRVLatencies.addSample(latency);
 	if (trLogInfo)
 		trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(
@@ -5069,11 +5074,18 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	return rep.version;
 }
 
+ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx) {
+	double t = cx->lastTimedGrv.get();
+	if (now() - t > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+		wait(cx->lastTimedGrv.whenAtLeast(now() - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
+	}
+	return cx->cachedRv;
+}
+
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
-		if (options.useGrvCache && cx->cachedReadVersion.isValid() && cx->cachedReadVersion.isReady() &&
-		    !cx->cachedReadVersion.isError()) {
-			return cx->cachedReadVersion;
+		if (options.useGrvCache && cx->cachedRv > Version(0)) {
+			return getDBCachedReadVersion(getDatabase().getPtr());
 		}
 		++cx->transactionReadVersions;
 		flags |= options.getReadVersionFlags;
@@ -5149,7 +5161,6 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		                                 startTime,
 		                                 metadataVersion,
 		                                 options.tags);
-		cx->updateCachedRV(readVersion);
 	}
 	return readVersion;
 }

From 87fac6e3016ee767af2b620cd31f9b5b5bfc32a1 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 13 Sep 2021 13:04:55 -0400
Subject: [PATCH 003/210] adjust to changed function signature after merge

---
 fdbclient/NativeAPI.actor.cpp      | 2 +-
 fdbclient/ReadYourWrites.actor.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 53796a8fbb..caa64b2fe9 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5548,7 +5548,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::USE_GRV_CACHE:
 		// Needs to be a no-op if ratekeeper is throttling
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 		options.useGrvCache = true;
 		break;
 
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index f1637f357e..b9fc27effd 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2250,7 +2250,7 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 		options.bypassUnreadable = true;
 		break;
 	case FDBTransactionOptions::USE_GRV_CACHE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 		options.useGrvCache = true;
 	default:
 		break;

From 4cd5d138ffbd1f64a43673c4475f361f0c0c1133 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 22 Sep 2021 13:13:35 -0400
Subject: [PATCH 004/210] Fixed segfault involving missing futures and infinite
 looping in background actor

---
 fdbclient/NativeAPI.actor.cpp | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index caa64b2fe9..bd53283d20 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -934,24 +934,26 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state double grvDelay = 0.001;
 	cx->lastTimedGrv = 0.0;
 	cx->cachedRv = Version(0);
-	loop {
-		wait(refreshTransaction(cx, &tr));
-		state double curTime = now();
-		state double lastTime = cx->lastTimedGrv.get();
-		if (curTime - lastTime > (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
-			try {
-				Version v = wait(tr.getReadVersion());
-				// Take the average of what we expect and what we got as the delay
-				grvDelay = (grvDelay + (now() - curTime)) / 2.0;
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw;
+	try {
+		loop {
+			wait(refreshTransaction(cx, &tr));
+			state double curTime = now();
+			state double lastTime = cx->lastTimedGrv.get();
+			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
+				try {
+					wait(success(tr.getReadVersion()));
+					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
+				} catch (Error& e) {
+					TraceEvent("BackgroundGrvUpdaterTxnError").error(e, true);
+					wait(tr.onError(e));
 				}
-				wait(tr.onError(e));
+			} else {
+				wait(delay(0.001 + (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)));
 			}
-		} else {
-			wait(delay((CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)));
 		}
+	} catch (Error& e) {
+		TraceEvent("BackgroundGrvUpdaterFailed").error(e, true);
+		throw;
 	}
 }
 
@@ -5767,7 +5769,8 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx) {
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (options.useGrvCache && cx->cachedRv > Version(0)) {
-			return getDBCachedReadVersion(getDatabase().getPtr());
+			readVersion = getDBCachedReadVersion(getDatabase().getPtr());
+			return readVersion;
 		}
 		++cx->transactionReadVersions;
 		flags |= options.getReadVersionFlags;

From d560eb1feafe2c46e2b64d54f38bc9d1f4d4ffd2 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 29 Sep 2021 13:45:23 -0400
Subject: [PATCH 005/210] debug time bounds using sim_validation

---
 fdbclient/NativeAPI.actor.cpp         | 22 ++++++---
 fdbrpc/sim_validation.cpp             | 69 +++++++++++++++++++++++++++
 fdbrpc/sim_validation.h               |  8 ++++
 fdbserver/CommitProxyServer.actor.cpp |  2 +
 fdbserver/GrvProxyServer.actor.cpp    |  1 +
 fdbserver/workloads/Cycle.actor.cpp   |  1 +
 6 files changed, 96 insertions(+), 7 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index bd53283d20..63183d8fcc 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -58,6 +58,7 @@
 #include "fdbrpc/LoadBalance.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
+#include "fdbrpc/sim_validation.h"
 #include "fdbserver/Knobs.h"
 #include "flow/Arena.h"
 #include "flow/ActorCollection.h"
@@ -195,9 +196,14 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 }
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (t >= lastTimedGrv.get()) {
-		lastTimedGrv = t;
+	if (t > lastTimedGrv.get()) {
+		// TraceEvent("CheckpointCacheUpdate")
+		// 	.detail("Version", v)
+		// 	.detail("CurTime", t)
+		// 	.detail("LastVersion", cachedRv)
+		// 	.detail("LastTime", lastTimedGrv.get());
 		cachedRv = v;
+		lastTimedGrv = t;
 	}
 }
 
@@ -5758,18 +5764,20 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	return rep.version;
 }
 
-ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx) {
-	double t = cx->lastTimedGrv.get();
-	if (now() - t > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
-		wait(cx->lastTimedGrv.whenAtLeast(now() - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
+ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double requestTime) {
+	if (requestTime - cx->lastTimedGrv.get() > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+		wait(cx->lastTimedGrv.whenAtLeast(requestTime - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
 	}
+	// Want to check that the cached version is at most
+	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
+	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
 	return cx->cachedRv;
 }
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (options.useGrvCache && cx->cachedRv > Version(0)) {
-			readVersion = getDBCachedReadVersion(getDatabase().getPtr());
+			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
 			return readVersion;
 		}
 		++cx->transactionReadVersions;
diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp
index a6322740ac..85cbb66a95 100644
--- a/fdbrpc/sim_validation.cpp
+++ b/fdbrpc/sim_validation.cpp
@@ -25,6 +25,7 @@
 
 // used for simulation validations
 static std::map<std::string, int64_t> validationData;
+static std::map<std::string, double> validationData2;
 static std::set<UID> disabledMachines;
 
 void debug_setVersionCheckEnabled(UID uid, bool enabled) {
@@ -127,3 +128,71 @@ bool debug_isCheckRelocationDuration() {
 void debug_setCheckRelocationDuration(bool check) {
 	checkRelocationDuration = check;
 }
+void debug_advanceVersionTimestamp(int64_t version, double minTime, double maxTime) {
+	debug_advanceVersionMinTimestamp(version, minTime);
+	debug_advanceVersionMaxTimestamp(version, maxTime);
+}
+
+void debug_advanceTime(int64_t version, double t, const char* suffix) {
+	auto& entry = validationData2[std::to_string(version) + suffix];
+	if (t > entry) {
+		entry = t;
+	}
+}
+
+void debug_advanceVersionMinTimestamp(int64_t version, double t) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return;
+	debug_advanceTime(version, t, "min");
+}
+
+void debug_advanceVersionMaxTimestamp(int64_t version, double t) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return;
+	debug_advanceTime(version, t, "max");
+}
+
+bool debug_checkPartVersionTime(int64_t version,
+                                double t,
+                                std::string context,
+                                std::string minormax,
+                                Severity sev = SevError) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return false;
+	if (!validationData2.count(std::to_string(version) + minormax)) {
+		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
+		    .detail("VersionChecking", version)
+		    .detail("TimeChecking", t);
+		return false;
+	}
+	int sign = minormax == "min" ? 1 : -1;
+	if (t * sign < validationData2[std::to_string(version) + minormax] * sign) {
+		TraceEvent(sev, (context + "DurabilityError").c_str())
+		    .detail("VersionChecking", version)
+		    .detail("TimeChecking", t)
+		    .detail("MinMaxChecking", minormax)
+		    .detail("MinTime", validationData2[std::to_string(version) + "min"])
+		    .detail("MaxTime", validationData2[std::to_string(version) + "max"]);
+		return true;
+	}
+	return false;
+}
+
+bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return false;
+	return debug_checkPartVersionTime(version, t, context, "min", sev) ||
+	       debug_checkPartVersionTime(version, t, context, "max", sev);
+}
+
+bool debug_checkVersionMinTime(int64_t version, double t, std::string context, Severity sev) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return false;
+	return debug_checkPartVersionTime(version, t, context, "min", sev);
+}
+
+bool debug_checkVersionMaxTime(int64_t version, double t, std::string context, Severity sev) {
+	if (!g_network->isSimulated() || g_simulator.extraDB)
+		return false;
+	return debug_checkPartVersionTime(version, t, context, "max", sev);
+}
\ No newline at end of file
diff --git a/fdbrpc/sim_validation.h b/fdbrpc/sim_validation.h
index 1d563aaa23..8c110ce898 100644
--- a/fdbrpc/sim_validation.h
+++ b/fdbrpc/sim_validation.h
@@ -48,4 +48,12 @@ bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context,
 bool debug_isCheckRelocationDuration();
 void debug_setCheckRelocationDuration(bool check);
 
+void debug_advanceVersionTimestamp(int64_t version, double minTime, double maxTime);
+void debug_advanceVersionMinTimestamp(int64_t version, double t);
+void debug_advanceVersionMaxTimestamp(int64_t version, double t);
+
+bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev = SevError);
+bool debug_checkVersionMinTime(int64_t version, double t, std::string context, Severity sev = SevError);
+bool debug_checkVersionMaxTime(int64_t version, double t, std::string context, Severity sev = SevError);
+
 #endif
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 413a4456e1..19ac018362 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -1187,6 +1187,8 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 
 	self->commitStartTime = now();
 	pProxyCommitData->lastStartCommit = self->commitStartTime;
+	auto curTime = now();
+	debug_advanceVersionTimestamp(self->commitVersion, curTime, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
 	self->loggingComplete = pProxyCommitData->logSystem->push(self->prevVersion,
 	                                                          self->commitVersion,
 	                                                          pProxyCommitData->committedVersion.get(),
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index bd88a7f402..aa214cd64b 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -26,6 +26,7 @@
 #include "fdbclient/GrvProxyInterface.h"
 #include "fdbserver/WaitFailure.h"
 #include "fdbserver/WorkerInterface.actor.h"
+#include "fdbrpc/sim_validation.h"
 #include "flow/flow.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp
index edaa18851f..3bd22df595 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.actor.cpp
@@ -132,6 +132,7 @@ struct CycleWorkload : TestWorkload {
 						// TraceEvent("CyclicTest").detail("Key", self->key(r2).toString()).detail("Value", self->value(r4).toString());
 						// TraceEvent("CyclicTest").detail("Key", self->key(r3).toString()).detail("Value", self->value(r2).toString());
 
+						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						wait(tr.commit());
 						// TraceEvent("CycleCommit");
 						break;

From ae31f01dc49d6e733a65b0c0d76e7b47ae37f6f4 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 5 Oct 2021 12:54:22 -0400
Subject: [PATCH 006/210] remove cache update from extractreadversion and set
 lower time bound to 0

---
 fdbclient/NativeAPI.actor.cpp         | 1 -
 fdbserver/CommitProxyServer.actor.cpp | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 63183d8fcc..5ba155e55f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5709,7 +5709,6 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	state Span span(spanContext, location, { parent });
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
-	cx->updateCachedRV(startTime, rep.version);
 	cx->GRVLatencies.addSample(latency);
 	if (trLogInfo)
 		trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 19ac018362..e1c6be6429 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -1188,7 +1188,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	self->commitStartTime = now();
 	pProxyCommitData->lastStartCommit = self->commitStartTime;
 	auto curTime = now();
-	debug_advanceVersionTimestamp(self->commitVersion, curTime, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
+	debug_advanceVersionTimestamp(self->commitVersion, 0, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
 	self->loggingComplete = pProxyCommitData->logSystem->push(self->prevVersion,
 	                                                          self->commitVersion,
 	                                                          pProxyCommitData->committedVersion.get(),

From 9a18cc8f41d027309fab428012dd94438baf42d2 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 5 Oct 2021 17:31:31 -0400
Subject: [PATCH 007/210] Change debug time bound update logic to be located in
 masterserver

---
 fdbclient/NativeAPI.actor.cpp         | 1 +
 fdbserver/CommitProxyServer.actor.cpp | 2 --
 fdbserver/masterserver.actor.cpp      | 6 ++++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a219d471d4..b3827a4ac3 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5703,6 +5703,7 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	state Span span(spanContext, location, { parent });
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
+	cx->updateCachedRV(startTime, rep.version);
 	cx->GRVLatencies.addSample(latency);
 	if (trLogInfo)
 		trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 56839e0325..fa755359ef 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -1187,8 +1187,6 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 
 	self->commitStartTime = now();
 	pProxyCommitData->lastStartCommit = self->commitStartTime;
-	auto curTime = now();
-	debug_advanceVersionTimestamp(self->commitVersion, 0, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
 	self->loggingComplete = pProxyCommitData->logSystem->push(self->prevVersion,
 	                                                          self->commitVersion,
 	                                                          pProxyCommitData->committedVersion.get(),
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 275ffdc140..39523e6e7c 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1249,6 +1249,12 @@ ACTOR Future<Void> serveLiveCommittedVersion(Reference<MasterData> self) {
 			         waitNext(self->myInterface.reportLiveCommittedVersion.getFuture())) {
 				self->minKnownCommittedVersion = std::max(self->minKnownCommittedVersion, req.minKnownCommittedVersion);
 				if (req.version > self->liveCommittedVersion) {
+					auto curTime = now();
+					// add debug here to change liveCommittedVersion to time bound of now()
+					debug_advanceVersionTimestamp(
+					    self->liveCommittedVersion, 0, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
+					// also add req.version but with no time bound
+					debug_advanceVersionTimestamp(req.version, 0, std::numeric_limits<double>::max());
 					self->liveCommittedVersion = req.version;
 					self->databaseLocked = req.locked;
 					self->proxyMetadataVersion = req.metadataVersion;

From 4c942cc4e3e51314d34422abbcd68931592e450a Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 6 Oct 2021 14:19:30 -0400
Subject: [PATCH 008/210] simplify sim_validation verification to only involve
 maximum bound

---
 fdbrpc/sim_validation.cpp        | 76 +++++++-------------------------
 fdbrpc/sim_validation.h          |  7 +--
 fdbserver/masterserver.actor.cpp |  6 +--
 3 files changed, 20 insertions(+), 69 deletions(-)

diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp
index 85cbb66a95..eb817b1195 100644
--- a/fdbrpc/sim_validation.cpp
+++ b/fdbrpc/sim_validation.cpp
@@ -25,7 +25,7 @@
 
 // used for simulation validations
 static std::map<std::string, int64_t> validationData;
-static std::map<std::string, double> validationData2;
+static std::map<int64_t, double> validationData2;
 static std::set<UID> disabledMachines;
 
 void debug_setVersionCheckEnabled(UID uid, bool enabled) {
@@ -128,71 +128,27 @@ bool debug_isCheckRelocationDuration() {
 void debug_setCheckRelocationDuration(bool check) {
 	checkRelocationDuration = check;
 }
-void debug_advanceVersionTimestamp(int64_t version, double minTime, double maxTime) {
-	debug_advanceVersionMinTimestamp(version, minTime);
-	debug_advanceVersionMaxTimestamp(version, maxTime);
-}
-
-void debug_advanceTime(int64_t version, double t, const char* suffix) {
-	auto& entry = validationData2[std::to_string(version) + suffix];
-	if (t > entry) {
-		entry = t;
-	}
-}
-
-void debug_advanceVersionMinTimestamp(int64_t version, double t) {
+void debug_advanceVersionTimestamp(int64_t version, double t) {
 	if (!g_network->isSimulated() || g_simulator.extraDB)
 		return;
-	debug_advanceTime(version, t, "min");
-}
-
-void debug_advanceVersionMaxTimestamp(int64_t version, double t) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
-		return;
-	debug_advanceTime(version, t, "max");
-}
-
-bool debug_checkPartVersionTime(int64_t version,
-                                double t,
-                                std::string context,
-                                std::string minormax,
-                                Severity sev = SevError) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
-		return false;
-	if (!validationData2.count(std::to_string(version) + minormax)) {
-		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
-		    .detail("VersionChecking", version)
-		    .detail("TimeChecking", t);
-		return false;
-	}
-	int sign = minormax == "min" ? 1 : -1;
-	if (t * sign < validationData2[std::to_string(version) + minormax] * sign) {
-		TraceEvent(sev, (context + "DurabilityError").c_str())
-		    .detail("VersionChecking", version)
-		    .detail("TimeChecking", t)
-		    .detail("MinMaxChecking", minormax)
-		    .detail("MinTime", validationData2[std::to_string(version) + "min"])
-		    .detail("MaxTime", validationData2[std::to_string(version) + "max"]);
-		return true;
-	}
-	return false;
+	validationData2[version] = t;
 }
 
 bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
 	if (!g_network->isSimulated() || g_simulator.extraDB)
 		return false;
-	return debug_checkPartVersionTime(version, t, context, "min", sev) ||
-	       debug_checkPartVersionTime(version, t, context, "max", sev);
-}
-
-bool debug_checkVersionMinTime(int64_t version, double t, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!validationData2.count(version)) {
+		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
+		    .detail("VersionChecking", version)
+		    .detail("TimeChecking", t);
 		return false;
-	return debug_checkPartVersionTime(version, t, context, "min", sev);
-}
-
-bool debug_checkVersionMaxTime(int64_t version, double t, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
-		return false;
-	return debug_checkPartVersionTime(version, t, context, "max", sev);
+	}
+	if (t > validationData2[version]) {
+		TraceEvent(sev, (context + "DurabilityError").c_str())
+		    .detail("VersionChecking", version)
+		    .detail("TimeChecking", t)
+		    .detail("MaxTime", validationData2[version]);
+		return true;
+	}
+	return false;
 }
\ No newline at end of file
diff --git a/fdbrpc/sim_validation.h b/fdbrpc/sim_validation.h
index 8c110ce898..975b6a424f 100644
--- a/fdbrpc/sim_validation.h
+++ b/fdbrpc/sim_validation.h
@@ -48,12 +48,7 @@ bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context,
 bool debug_isCheckRelocationDuration();
 void debug_setCheckRelocationDuration(bool check);
 
-void debug_advanceVersionTimestamp(int64_t version, double minTime, double maxTime);
-void debug_advanceVersionMinTimestamp(int64_t version, double t);
-void debug_advanceVersionMaxTimestamp(int64_t version, double t);
-
+void debug_advanceVersionTimestamp(int64_t version, double t);
 bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev = SevError);
-bool debug_checkVersionMinTime(int64_t version, double t, std::string context, Severity sev = SevError);
-bool debug_checkVersionMaxTime(int64_t version, double t, std::string context, Severity sev = SevError);
 
 #endif
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 39523e6e7c..2e6452cfb6 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1251,10 +1251,10 @@ ACTOR Future<Void> serveLiveCommittedVersion(Reference<MasterData> self) {
 				if (req.version > self->liveCommittedVersion) {
 					auto curTime = now();
 					// add debug here to change liveCommittedVersion to time bound of now()
-					debug_advanceVersionTimestamp(
-					    self->liveCommittedVersion, 0, curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
+					debug_advanceVersionTimestamp(self->liveCommittedVersion,
+					                              curTime + CLIENT_KNOBS->MAX_VERSION_CACHE_LAG);
 					// also add req.version but with no time bound
-					debug_advanceVersionTimestamp(req.version, 0, std::numeric_limits<double>::max());
+					debug_advanceVersionTimestamp(req.version, std::numeric_limits<double>::max());
 					self->liveCommittedVersion = req.version;
 					self->databaseLocked = req.locked;
 					self->proxyMetadataVersion = req.metadataVersion;

From cc38c48123a8369627af2b95ef8496af343af7de Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 6 Oct 2021 14:35:42 -0400
Subject: [PATCH 009/210] Add debug knob to use grv cache for performance
 testing

---
 fdbclient/ClientKnobs.cpp     | 1 +
 fdbclient/ClientKnobs.h       | 1 +
 fdbclient/NativeAPI.actor.cpp | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index f96e6ab37d..797130d905 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -114,6 +114,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( LOG_RANGE_BLOCK_SIZE, CORE_VERSIONSPERSECOND );
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
+	init( DEBUG_USE_GRV_CACHE,                    false );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 23d598e07f..d557088131 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -117,6 +117,7 @@ public:
 	int LOG_RANGE_BLOCK_SIZE;
 	int MUTATION_BLOCK_SIZE;
 	double MAX_VERSION_CACHE_LAG;
+	bool DEBUG_USE_GRV_CACHE; // Turn on to have all GRVs request to use the cache
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b3827a4ac3..7b87b9ce6c 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5770,7 +5770,7 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double request
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
-		if (options.useGrvCache && cx->cachedRv > Version(0)) {
+		if ((CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache) && cx->cachedRv > Version(0)) {
 			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
 			return readVersion;
 		}

From ecdafba3367e9b09fcb313169da662f80413d7ea Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 8 Oct 2021 15:17:28 -0400
Subject: [PATCH 010/210] add debug traces to background grv updater

---
 fdbclient/NativeAPI.actor.cpp       | 15 ++++++++++++++-
 fdbserver/workloads/Cycle.actor.cpp |  2 +-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 7b87b9ce6c..4a14a09950 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -193,7 +193,7 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 }
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (t > lastTimedGrv.get()) {
+	if (t > lastTimedGrv.get() && v >= cachedRv) {
 		// TraceEvent("CheckpointCacheUpdate")
 		// 	.detail("Version", v)
 		// 	.detail("CurTime", t)
@@ -937,15 +937,28 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state double grvDelay = 0.001;
 	cx->lastTimedGrv = 0.0;
 	cx->cachedRv = Version(0);
+	TraceEvent(SevDebug, "BackgroundGrvUpdaterStart").detail("DBID", cx->dbId).detail("GrvDelay", grvDelay);
 	try {
 		loop {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
 			state double lastTime = cx->lastTimedGrv.get();
+			TraceEvent(SevDebug, "BackgroundGrvUpdaterBefore")
+			    .detail("CurTime", curTime)
+			    .detail("LastTime", lastTime)
+			    .detail("GrvDelay", grvDelay)
+			    .detail("CachedRv", cx->cachedRv)
+			    .detail("CachedTime", cx->lastTimedGrv.get())
+			    .detail("Gap", curTime - lastTime)
+			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
 			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
 				try {
 					wait(success(tr.getReadVersion()));
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
+					TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
+					    .detail("GrvDelay", grvDelay)
+					    .detail("CachedRv", cx->cachedRv)
+					    .detail("CachedTime", cx->lastTimedGrv.get());
 				} catch (Error& e) {
 					TraceEvent("BackgroundGrvUpdaterTxnError").error(e, true);
 					wait(tr.onError(e));
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp
index cc3d2eecf6..74623fdacb 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.actor.cpp
@@ -110,6 +110,7 @@ struct CycleWorkload : TestWorkload {
 				}
 				while (true) {
 					try {
+						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						// Reverse next and next^2 node
 						Optional<Value> v = wait(tr.get(self->key(r)));
 						if (!v.present())
@@ -132,7 +133,6 @@ struct CycleWorkload : TestWorkload {
 						// TraceEvent("CyclicTest").detail("Key", self->key(r2).toString()).detail("Value", self->value(r4).toString());
 						// TraceEvent("CyclicTest").detail("Key", self->key(r3).toString()).detail("Value", self->value(r2).toString());
 
-						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						wait(tr.commit());
 						// TraceEvent("CycleCommit");
 						break;

From b770936cde3396bb653e0202d8a37ce8d1bc7e13 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 8 Oct 2021 16:38:21 -0400
Subject: [PATCH 011/210] move background updater actor initialization

---
 fdbclient/DatabaseContext.h   | 10 +++++-----
 fdbclient/NativeAPI.actor.cpp | 18 +++++++++++-------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 3d95bbf67d..c1f144ef8a 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -330,11 +330,6 @@ public:
 	// map from tssid -> metrics for that tss pair
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 
-	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
-	NotifiedDouble lastTimedGrv;
-	Version cachedRv;
-	void updateCachedRV(double t, Version v);
-
 	UID dbId;
 	IsInternal internal; // Only contexts created through the C client and fdbcli are non-internal
 
@@ -389,6 +384,11 @@ public:
 	int outstandingWatches;
 	int maxOutstandingWatches;
 
+	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
+	NotifiedDouble lastTimedGrv;
+	Version cachedRv;
+	void updateCachedRV(double t, Version v);
+
 	int snapshotRywEnabled;
 
 	int transactionTracingEnabled;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4a14a09950..38d2f1c197 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -935,8 +935,6 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state Transaction tr;
 	state double grvDelay = 0.001;
-	cx->lastTimedGrv = 0.0;
-	cx->cachedRv = Version(0);
 	TraceEvent(SevDebug, "BackgroundGrvUpdaterStart").detail("DBID", cx->dbId).detail("GrvDelay", grvDelay);
 	try {
 		loop {
@@ -1199,9 +1197,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
     commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0),
-    transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
-    coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
-    detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
+    lastTimedGrv(0.0), cachedRv(0), transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo),
+    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
+    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
 	connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size())
@@ -1222,7 +1221,6 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 
 	monitorProxiesInfoChange = monitorProxiesChange(clientInfo, &proxiesChangeTrigger);
 	tssMismatchHandler = handleTssMismatches(this);
-	grvUpdateHandler = backgroundGrvUpdater(this);
 	clientStatusUpdater.actor = clientStatusUpdateActor(this);
 	cacheListMonitor = monitorCacheList(this);
 
@@ -1482,7 +1480,9 @@ DatabaseContext::~DatabaseContext() {
 	monitorProxiesInfoChange.cancel();
 	monitorTssInfoChange.cancel();
 	tssMismatchHandler.cancel();
-	grvUpdateHandler.cancel();
+	if (grvUpdateHandler.isValid()) {
+		grvUpdateHandler.cancel();
+	}
 	for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it))
 		it->second->notifyContextDestroyed();
 	ASSERT_ABORT(server_interf.empty());
@@ -5784,6 +5784,10 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double request
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if ((CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache) && cx->cachedRv > Version(0)) {
+			// Upon our first request to use cached RVs, start the background updater
+			if (!cx->grvUpdateHandler.isValid()) {
+				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
+			}
 			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
 			return readVersion;
 		}

From 2b2d059039cb7c3f155a4e1e44b5aa773203a381 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 8 Oct 2021 17:25:07 -0400
Subject: [PATCH 012/210] Specify option to specifically skip grv cache for the
 background updater when using debug knob

---
 fdbclient/NativeAPI.actor.cpp        | 9 ++++++++-
 fdbclient/NativeAPI.actor.h          | 1 +
 fdbclient/ReadYourWrites.actor.cpp   | 3 +++
 fdbclient/ReadYourWrites.h           | 1 +
 fdbclient/vexillographer/fdb.options | 2 ++
 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 38d2f1c197..8d3caba4e1 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -951,6 +951,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
 			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
 				try {
+					tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
 					wait(success(tr.getReadVersion()));
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
 					TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
@@ -4789,6 +4790,7 @@ void TransactionOptions::clear() {
 	priority = TransactionPriority::DEFAULT;
 	expensiveClearCostEstimation = false;
 	useGrvCache = false;
+	skipGrvCache = false;
 }
 
 TransactionOptions::TransactionOptions() {
@@ -5567,6 +5569,11 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		options.useGrvCache = true;
 		break;
 
+	case FDBTransactionOptions::SKIP_GRV_CACHE:
+		validateOptionValueNotPresent(value);
+		options.skipGrvCache = true;
+		break;
+
 	default:
 		break;
 	}
@@ -5783,7 +5790,7 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double request
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
-		if ((CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache) && cx->cachedRv > Version(0)) {
+		if (!options.skipGrvCache && (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index f986680abe..7a3d8ccc3d 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -154,6 +154,7 @@ struct TransactionOptions {
 	bool reportConflictingKeys : 1;
 	bool expensiveClearCostEstimation : 1;
 	bool useGrvCache : 1;
+	bool skipGrvCache : 1;
 
 	TransactionPriority priority;
 
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index d407f88bf7..005a398ade 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2252,6 +2252,9 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 	case FDBTransactionOptions::USE_GRV_CACHE:
 		validateOptionValueNotPresent(value);
 		options.useGrvCache = true;
+	case FDBTransactionOptions::SKIP_GRV_CACHE:
+		validateOptionValueNotPresent(value);
+		options.skipGrvCache = true;
 	default:
 		break;
 	}
diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h
index 6ab02e4e61..109ab0663c 100644
--- a/fdbclient/ReadYourWrites.h
+++ b/fdbclient/ReadYourWrites.h
@@ -45,6 +45,7 @@ struct ReadYourWritesTransactionOptions {
 	int snapshotRywEnabled;
 	bool bypassUnreadable : 1;
 	bool useGrvCache : 1;
+	bool skipGrvCache : 1;
 
 	ReadYourWritesTransactionOptions() {}
 	explicit ReadYourWritesTransactionOptions(Transaction const& tr);
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index bdadbed577..6a65c35ec0 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -296,6 +296,8 @@ description is not currently required but encouraged.
                 description="Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp." />            
     <Option name="use_grv_cache" code="1101"
             description="Allows this transaction to use cached GRV from the database context." />
+    <Option name="skip_grv_cache" code="1102"
+            description="Specifically instruct this transaction to NOT use cached GRV." />
   </Scope>
 
   <!-- The enumeration values matter - do not change them without

From f75c8b25adc57ef07aaa8c1969fa0e86263741e4 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 18 Oct 2021 13:59:04 -0400
Subject: [PATCH 013/210] add small random delay before returning a cached read
 version

---
 fdbclient/NativeAPI.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 8d3caba4e1..e40b2256ad 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5785,6 +5785,7 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double request
 	// Want to check that the cached version is at most
 	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
 	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
+	wait(delay(0.001 * deterministicRandom()->random01()));
 	return cx->cachedRv;
 }
 

From f1c8d3fbc846ed0efc63bc2579341e7a4f8ffe0b Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 19 Oct 2021 17:59:30 -0400
Subject: [PATCH 014/210] Add code to disable cache when ratekeeper begins
 throttling

---
 fdbclient/ClientKnobs.cpp           |  1 +
 fdbclient/ClientKnobs.h             |  1 +
 fdbclient/CommitProxyInterface.h    |  4 +++-
 fdbclient/DatabaseContext.h         |  2 ++
 fdbclient/NativeAPI.actor.cpp       | 21 ++++++++++++++++-----
 fdbserver/GrvProxyServer.actor.cpp  |  6 ++++++
 fdbserver/workloads/Cycle.actor.cpp |  2 +-
 7 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 797130d905..7057245e8d 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -115,6 +115,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( DEBUG_USE_GRV_CACHE,                    false );
+	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index d557088131..1aeb4dcb2c 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -118,6 +118,7 @@ public:
 	int MUTATION_BLOCK_SIZE;
 	double MAX_VERSION_CACHE_LAG;
 	bool DEBUG_USE_GRV_CACHE; // Turn on to have all GRVs request to use the cache
+	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index ecb745d318..f73331f469 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -195,6 +195,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	bool locked;
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
+	bool ratekeeperThrottling;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
@@ -208,7 +209,8 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 		           locked,
 		           metadataVersion,
 		           tagThrottleInfo,
-		           midShardSize);
+		           midShardSize,
+		           ratekeeperThrottling);
 	}
 };
 
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index c1f144ef8a..46efcb3da9 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -384,10 +384,12 @@ public:
 	int outstandingWatches;
 	int maxOutstandingWatches;
 
+	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
 	NotifiedDouble lastTimedGrv;
 	Version cachedRv;
 	void updateCachedRV(double t, Version v);
+	double lastTimedRkThrottle;
 
 	int snapshotRywEnabled;
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index e40b2256ad..17bcf8ad5d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1198,9 +1198,9 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
     commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0),
-    lastTimedGrv(0.0), cachedRv(0), transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo),
-    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0), transactionTracingEnabled(true), taskID(taskID),
+    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
+    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -5724,6 +5724,10 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
 	cx->updateCachedRV(startTime, rep.version);
+	// use startTime instead??
+	if (rep.ratekeeperThrottling) {
+		cx->lastTimedRkThrottle = now();
+	}
 	cx->GRVLatencies.addSample(latency);
 	if (trLogInfo)
 		trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(
@@ -5785,13 +5789,20 @@ ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double request
 	// Want to check that the cached version is at most
 	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
 	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
-	wait(delay(0.001 * deterministicRandom()->random01()));
 	return cx->cachedRv;
 }
 
+bool rkThrottlingCooledDown(DatabaseContext* cx) {
+	if (cx->lastTimedRkThrottle == 0.0) {
+		return true;
+	}
+	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
+}
+
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
-		if (!options.skipGrvCache && (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
+		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
+		    (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index f608b8f331..ab94b6299b 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -566,6 +566,12 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanID parentSpan,
 
 	grvProxyData->stats.grvGetCommittedVersionRpcDist->sampleSeconds(now() - grvConfirmEpochLive);
 	GetReadVersionReply rep;
+	rep.ratekeeperThrottling = false;
+	// borrowed logic from line 410-411 about hitting limit
+	if (grvProxyData->stats.txnRequestIn.getValue() - grvProxyData->stats.txnRequestOut.getValue() >
+	    SERVER_KNOBS->START_TRANSACTION_MAX_QUEUE_SIZE) {
+		rep.ratekeeperThrottling = true;
+	}
 	rep.version = repFromMaster.version;
 	rep.locked = repFromMaster.locked;
 	rep.metadataVersion = repFromMaster.metadataVersion;
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp
index 74623fdacb..d4de002a9e 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.actor.cpp
@@ -110,7 +110,6 @@ struct CycleWorkload : TestWorkload {
 				}
 				while (true) {
 					try {
-						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						// Reverse next and next^2 node
 						Optional<Value> v = wait(tr.get(self->key(r)));
 						if (!v.present())
@@ -126,6 +125,7 @@ struct CycleWorkload : TestWorkload {
 						int r4 = self->fromValue(v3.get());
 
 						tr.clear(self->key(r)); //< Shouldn't have an effect, but will break with wrong ordering
+						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						tr.set(self->key(r), self->value(r3));
 						tr.set(self->key(r2), self->value(r4));
 						tr.set(self->key(r3), self->value(r2));

From 6d742397607ebd450b41925ce9b4e8d334ef82f1 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 21 Oct 2021 16:44:10 -0400
Subject: [PATCH 015/210] Track throttling by measuring time spent left in
 queue on the proxy

---
 fdbclient/ClientKnobs.cpp          |  1 +
 fdbclient/ClientKnobs.h            |  2 ++
 fdbclient/CommitProxyInterface.h   | 13 +++++++------
 fdbclient/NativeAPI.actor.cpp      |  6 ++++--
 fdbserver/GrvProxyServer.actor.cpp | 18 ++++++++++++------
 5 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 7057245e8d..7eb6cf56a0 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -116,6 +116,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( DEBUG_USE_GRV_CACHE,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
+	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 1aeb4dcb2c..e9e70ce0ea 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -119,6 +119,8 @@ public:
 	double MAX_VERSION_CACHE_LAG;
 	bool DEBUG_USE_GRV_CACHE; // Turn on to have all GRVs request to use the cache
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
+	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // Adjust what amount of time is considered "sustained" throttling on
+	                                           // proxy for a GRV
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index f73331f469..a3531da863 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -195,7 +195,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	bool locked;
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
-	bool ratekeeperThrottling;
+	double timeThrottled;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
@@ -210,7 +210,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 		           metadataVersion,
 		           tagThrottleInfo,
 		           midShardSize,
-		           ratekeeperThrottling);
+		           timeThrottled);
 	}
 };
 
@@ -233,21 +233,22 @@ struct GetReadVersionRequest : TimedRequest {
 	uint32_t transactionCount;
 	uint32_t flags;
 	TransactionPriority priority;
+	double reqProcessStart, reqProcessEnd;
 
 	TransactionTagMap<uint32_t> tags;
 
 	Optional<UID> debugID;
 	ReplyPromise<GetReadVersionReply> reply;
 
-	GetReadVersionRequest() : transactionCount(1), flags(0) {}
+	GetReadVersionRequest() : transactionCount(1), flags(0), reqProcessStart(0.0), reqProcessEnd(0.0) {}
 	GetReadVersionRequest(SpanID spanContext,
 	                      uint32_t transactionCount,
 	                      TransactionPriority priority,
 	                      uint32_t flags = 0,
 	                      TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
 	                      Optional<UID> debugID = Optional<UID>())
-	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags),
-	    debugID(debugID) {
+	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority),
+	    reqProcessStart(0.0), reqProcessEnd(0.0), tags(tags), debugID(debugID) {
 		flags = flags & ~FLAG_PRIORITY_MASK;
 		switch (priority) {
 		case TransactionPriority::BATCH:
@@ -268,7 +269,7 @@ struct GetReadVersionRequest : TimedRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext);
+		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, reqProcessStart, reqProcessEnd);
 
 		if (ar.isDeserializing) {
 			if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 17bcf8ad5d..a200971fd8 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5724,8 +5724,10 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
 	cx->updateCachedRV(startTime, rep.version);
-	// use startTime instead??
-	if (rep.ratekeeperThrottling) {
+	// use startTime instead?
+	// maybe this also requires tracking number of loops processed in queue?
+	// TraceEvent("DebugTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
+	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
 		cx->lastTimedRkThrottle = now();
 	}
 	cx->GRVLatencies.addSample(latency);
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index ab94b6299b..beab806a2b 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -566,12 +566,6 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanID parentSpan,
 
 	grvProxyData->stats.grvGetCommittedVersionRpcDist->sampleSeconds(now() - grvConfirmEpochLive);
 	GetReadVersionReply rep;
-	rep.ratekeeperThrottling = false;
-	// borrowed logic from line 410-411 about hitting limit
-	if (grvProxyData->stats.txnRequestIn.getValue() - grvProxyData->stats.txnRequestOut.getValue() >
-	    SERVER_KNOBS->START_TRANSACTION_MAX_QUEUE_SIZE) {
-		rep.ratekeeperThrottling = true;
-	}
 	rep.version = repFromMaster.version;
 	rep.locked = repFromMaster.locked;
 	rep.metadataVersion = repFromMaster.metadataVersion;
@@ -654,6 +648,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			}
 		}
 
+		reply.timeThrottled = request.reqProcessEnd - request.reqProcessStart;
 		request.reply.send(reply);
 		++stats->txnRequestOut;
 	}
@@ -819,11 +814,21 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 
 			if (req.priority < TransactionPriority::DEFAULT &&
 			    !batchRateInfo.canStart(transactionsStarted[0] + transactionsStarted[1], tc)) {
+				if (req.reqProcessStart == 0.0) {
+					req.reqProcessStart = g_network->timer();
+				}
 				break;
 			} else if (req.priority < TransactionPriority::IMMEDIATE &&
 			           !normalRateInfo.canStart(transactionsStarted[0] + transactionsStarted[1], tc)) {
+				if (req.reqProcessStart == 0.0) {
+					req.reqProcessStart = g_network->timer();
+				}
 				break;
 			}
+			// If not previously set, this is the first time this transaction is being processed.
+			if (req.reqProcessStart == 0.0) {
+				req.reqProcessStart = g_network->timer();
+			}
 
 			if (req.debugID.present()) {
 				if (!debugID.present())
@@ -846,6 +851,7 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 				--grvProxyData->stats.batchGRVQueueSize;
 			}
 
+			req.reqProcessEnd = g_network->timer();
 			start[req.flags & 1].push_back(std::move(req));
 			static_assert(GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY == 1, "Implementation dependent on flag value");
 			transactionQueue->pop_front();

From 7273d352a265c7e7cda849888553f513f691dff5 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 25 Oct 2021 13:44:45 -0400
Subject: [PATCH 016/210] Add extra debug traces and ignore "IMMEDIATE"
 priority txns for rk throttling

---
 fdbclient/ClientKnobs.cpp     |  2 +-
 fdbclient/NativeAPI.actor.cpp | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 7eb6cf56a0..40b58375bd 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -116,7 +116,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( DEBUG_USE_GRV_CACHE,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
-	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
+	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,      0.01 );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a200971fd8..0697bbed13 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -66,6 +66,7 @@
 #include "flow/DeterministicRandom.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
+#include "flow/Trace.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Knobs.h"
@@ -5726,8 +5727,9 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	cx->updateCachedRV(startTime, rep.version);
 	// use startTime instead?
 	// maybe this also requires tracking number of loops processed in queue?
-	// TraceEvent("DebugTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
-	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+	TraceEvent(SevDebug, "DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
+	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD &&
+	    priority != TransactionPriority::IMMEDIATE) {
 		cx->lastTimedRkThrottle = now();
 	}
 	cx->GRVLatencies.addSample(latency);
@@ -5798,6 +5800,9 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 	if (cx->lastTimedRkThrottle == 0.0) {
 		return true;
 	}
+	TraceEvent(SevDebug, "DebugGrvRkCd")
+	    .detail("TimeElapsed", now() - cx->lastTimedRkThrottle)
+	    .detail("CooldownTime", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 }
 
@@ -5805,6 +5810,9 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
 		    (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
+			TraceEvent(SevDebug, "DebugGrvUseCache")
+			    .detail("LastRV", cx->cachedRv)
+			    .detail("LastTime", cx->lastTimedGrv.get());
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());

From e984b2f5d09a159266337e131b07e7524475f0b4 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 9 Nov 2021 15:13:50 -0500
Subject: [PATCH 017/210] remove sevdebug temporarily and add extra trace to
 verify knob values

---
 fdbclient/NativeAPI.actor.cpp | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 0697bbed13..21572a868d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -936,13 +936,19 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state Transaction tr;
 	state double grvDelay = 0.001;
-	TraceEvent(SevDebug, "BackgroundGrvUpdaterStart").detail("DBID", cx->dbId).detail("GrvDelay", grvDelay);
+	TraceEvent("BackgroundGrvUpdaterStart").detail("DBID", cx->dbId).detail("GrvDelay", grvDelay);
+	TraceEvent("DebugGrvCheckKnobValues")
+	    .detail("MinTraceSeverity", FLOW_KNOBS->MIN_TRACE_SEVERITY)
+	    .detail("GrvSustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD)
+	    .detail("GrvCacheRkCooldown", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN)
+	    .detail("MaxVersionCacheLag", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG)
+	    .detail("DebugUseGrvCache", CLIENT_KNOBS->DEBUG_USE_GRV_CACHE);
 	try {
 		loop {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
 			state double lastTime = cx->lastTimedGrv.get();
-			TraceEvent(SevDebug, "BackgroundGrvUpdaterBefore")
+			TraceEvent("BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
 			    .detail("GrvDelay", grvDelay)
@@ -955,7 +961,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
 					wait(success(tr.getReadVersion()));
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
-					TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
+					TraceEvent("BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
 					    .detail("CachedRv", cx->cachedRv)
 					    .detail("CachedTime", cx->lastTimedGrv.get());
@@ -5727,7 +5733,7 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	cx->updateCachedRV(startTime, rep.version);
 	// use startTime instead?
 	// maybe this also requires tracking number of loops processed in queue?
-	TraceEvent(SevDebug, "DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
+	TraceEvent("DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
 	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD &&
 	    priority != TransactionPriority::IMMEDIATE) {
 		cx->lastTimedRkThrottle = now();
@@ -5800,7 +5806,7 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 	if (cx->lastTimedRkThrottle == 0.0) {
 		return true;
 	}
-	TraceEvent(SevDebug, "DebugGrvRkCd")
+	TraceEvent("DebugGrvRkCd")
 	    .detail("TimeElapsed", now() - cx->lastTimedRkThrottle)
 	    .detail("CooldownTime", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
@@ -5810,9 +5816,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
 		    (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
-			TraceEvent(SevDebug, "DebugGrvUseCache")
-			    .detail("LastRV", cx->cachedRv)
-			    .detail("LastTime", cx->lastTimedGrv.get());
+			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", cx->lastTimedGrv.get());
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());

From 416d94d5b02e39076658d0b289f299bcd46f4672 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 15 Nov 2021 13:45:23 -0500
Subject: [PATCH 018/210] add some more trace details and remove spammy trace
 throttling temporarily

---
 fdbclient/NativeAPI.actor.cpp |  6 +++++-
 flow/Trace.cpp                | 20 ++++++++++----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 21572a868d..102437a148 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5736,6 +5736,9 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	TraceEvent("DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
 	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD &&
 	    priority != TransactionPriority::IMMEDIATE) {
+		TraceEvent("DebugGrvThrottleObserved")
+			.detail("TimeThrottled", rep.timeThrottled)
+			.detail("Threshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
 		cx->lastTimedRkThrottle = now();
 	}
 	cx->GRVLatencies.addSample(latency);
@@ -5807,6 +5810,7 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 		return true;
 	}
 	TraceEvent("DebugGrvRkCd")
+		.detail("LastTimedRkThrottle", cx->lastTimedRkThrottle)
 	    .detail("TimeElapsed", now() - cx->lastTimedRkThrottle)
 	    .detail("CooldownTime", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
@@ -5816,7 +5820,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
 		    (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
-			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", cx->lastTimedGrv.get());
+			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", format("%.6f", cx->lastTimedGrv.get()));
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 4a26f29cb2..bd830e47d3 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -911,16 +911,16 @@ bool TraceEvent::init() {
 	enabled = enabled && (!g_network || severity >= FLOW_KNOBS->MIN_TRACE_SEVERITY);
 
 	// Backstop to throttle very spammy trace events
-	if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
-		if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
-			enabled = false;
-			TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
-			    .suppressFor(5);
-		} else {
-			traceEventThrottlerCache->addAndExpire(
-			    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
-		}
-	}
+	// if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
+	// 	if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
+	// 		enabled = false;
+	// 		TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
+	// 		    .suppressFor(5);
+	// 	} else {
+	// 		traceEventThrottlerCache->addAndExpire(
+	// 		    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
+	// 	}
+	// }
 
 	if (enabled) {
 		tmpEventMetric = std::make_unique<DynamicEventMetric>(MetricNameRef());

From 508debd606cab925be5f095b07e5d64c1ef5d805 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 16 Nov 2021 13:08:23 -0500
Subject: [PATCH 019/210] increase trace log roll size (and max size)
 temporarily

---
 flow/Trace.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Trace.h b/flow/Trace.h
index aeaabb4373..a3da1cb790 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -33,7 +33,7 @@
 #include "flow/Error.h"
 #include "flow/ITrace.h"
 
-#define TRACE_DEFAULT_ROLL_SIZE (10 << 20)
+#define TRACE_DEFAULT_ROLL_SIZE (10 << 40)
 #define TRACE_DEFAULT_MAX_LOGS_SIZE (10 * TRACE_DEFAULT_ROLL_SIZE)
 
 inline int fastrand() {

From 16e551e83456018642b60a079afbbf7017ae3af9 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 16 Nov 2021 13:41:19 -0500
Subject: [PATCH 020/210] change knob to be an adjustable probability instead
 of binary on/off

---
 fdbclient/ClientKnobs.cpp     |  2 +-
 fdbclient/ClientKnobs.h       |  2 +-
 fdbclient/NativeAPI.actor.cpp | 13 +++++++------
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 40b58375bd..a20ec1b1b4 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -114,7 +114,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( LOG_RANGE_BLOCK_SIZE, CORE_VERSIONSPERSECOND );
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
-	init( DEBUG_USE_GRV_CACHE,                    false );
+	init( DEBUG_USE_GRV_CACHE_CHANCE,              -0.1 ); // Since we check <= for 100% chance at 1.0, we don't want the default to be 0. This also means 0 is not 0%.
 	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,      0.01 );
 
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index e9e70ce0ea..acc02ae76e 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -117,7 +117,7 @@ public:
 	int LOG_RANGE_BLOCK_SIZE;
 	int MUTATION_BLOCK_SIZE;
 	double MAX_VERSION_CACHE_LAG;
-	bool DEBUG_USE_GRV_CACHE; // Turn on to have all GRVs request to use the cache
+	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // Adjust what amount of time is considered "sustained" throttling on
 	                                           // proxy for a GRV
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 102437a148..54699f83a5 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -195,11 +195,11 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
 	if (t > lastTimedGrv.get() && v >= cachedRv) {
-		// TraceEvent("CheckpointCacheUpdate")
-		// 	.detail("Version", v)
-		// 	.detail("CurTime", t)
-		// 	.detail("LastVersion", cachedRv)
-		// 	.detail("LastTime", lastTimedGrv.get());
+		TraceEvent("CheckpointCacheUpdate")
+		    .detail("Version", v)
+		    .detail("CurTime", t)
+		    .detail("LastVersion", cachedRv)
+		    .detail("LastTime", lastTimedGrv.get());
 		cachedRv = v;
 		lastTimedGrv = t;
 	}
@@ -5145,6 +5145,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
+				TraceEvent("DebugGrvCommitSuccess");
 				cx->updateCachedRV(grvTime, v);
 				if (v != invalidVersion) {
 					if (CLIENT_BUGGIFY) {
@@ -5819,7 +5820,7 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
-		    (CLIENT_KNOBS->DEBUG_USE_GRV_CACHE || options.useGrvCache)) {
+		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache)) {
 			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", format("%.6f", cx->lastTimedGrv.get()));
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {

From 87144803707683c6d4cc16948b083c0f165c77ce Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 16 Nov 2021 13:54:05 -0500
Subject: [PATCH 021/210] fix some compile issues

---
 fdbclient/NativeAPI.actor.cpp | 2 +-
 flow/Trace.h                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 54699f83a5..a670cf84f6 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -942,7 +942,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	    .detail("GrvSustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD)
 	    .detail("GrvCacheRkCooldown", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN)
 	    .detail("MaxVersionCacheLag", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG)
-	    .detail("DebugUseGrvCache", CLIENT_KNOBS->DEBUG_USE_GRV_CACHE);
+	    .detail("DebugUseGrvCacheChance", CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE);
 	try {
 		loop {
 			wait(refreshTransaction(cx, &tr));
diff --git a/flow/Trace.h b/flow/Trace.h
index a3da1cb790..a7db8f2346 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -33,7 +33,7 @@
 #include "flow/Error.h"
 #include "flow/ITrace.h"
 
-#define TRACE_DEFAULT_ROLL_SIZE (10 << 40)
+#define TRACE_DEFAULT_ROLL_SIZE (10 << 24)
 #define TRACE_DEFAULT_MAX_LOGS_SIZE (10 * TRACE_DEFAULT_ROLL_SIZE)
 
 inline int fastrand() {

From 1e2060a52bd2b798da9a296e169cf47f59710a3c Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 17 Nov 2021 15:16:24 -0500
Subject: [PATCH 022/210] add extra tracking in dbcontext to periodically
 contact GRV proxies

---
 fdbclient/ClientKnobs.cpp     |  1 +
 fdbclient/ClientKnobs.h       |  5 +++--
 fdbclient/DatabaseContext.h   |  4 ++++
 fdbclient/NativeAPI.actor.cpp | 12 ++++++++----
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index a20ec1b1b4..bb7940ce47 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -114,6 +114,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( LOG_RANGE_BLOCK_SIZE, CORE_VERSIONSPERSECOND );
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
+	init( MAX_PROXY_CONTACT_LAG,                    1.0 );
 	init( DEBUG_USE_GRV_CACHE_CHANCE,              -0.1 ); // Since we check <= for 100% chance at 1.0, we don't want the default to be 0. This also means 0 is not 0%.
 	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,      0.01 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index acc02ae76e..c48c9cc8e8 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -116,11 +116,12 @@ public:
 	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
 	int LOG_RANGE_BLOCK_SIZE;
 	int MUTATION_BLOCK_SIZE;
-	double MAX_VERSION_CACHE_LAG;
+	double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
+	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // Adjust what amount of time is considered "sustained" throttling on
-	                                           // proxy for a GRV
+	                                           // proxy for a GRV which leads to disabling the cache
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 46efcb3da9..eea2b720bc 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -390,6 +390,10 @@ public:
 	Version cachedRv;
 	void updateCachedRV(double t, Version v);
 	double lastTimedRkThrottle;
+	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
+	// Because our checks for ratekeeper throttling requires communication with the proxies,
+	// we want to track the last time in order to periodically contact the proxy to check for throttling
+	double lastProxyRequest;
 
 	int snapshotRywEnabled;
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a670cf84f6..ae8ad4fd9a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -948,6 +948,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
 			state double lastTime = cx->lastTimedGrv.get();
+			state double lastProxyTime = cx->lastProxyRequest;
 			TraceEvent("BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
@@ -956,10 +957,12 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 			    .detail("CachedTime", cx->lastTimedGrv.get())
 			    .detail("Gap", curTime - lastTime)
 			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
-			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay)) {
+			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) ||
+			    curTime - lastProxyTime > CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG) {
 				try {
 					tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
 					wait(success(tr.getReadVersion()));
+					cx->lastProxyRequest = curTime;
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
 					TraceEvent("BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
@@ -1205,9 +1208,9 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
     commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0),
-    lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0), transactionTracingEnabled(true), taskID(taskID),
-    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
-    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0), lastProxyRequest(0.0), transactionTracingEnabled(true),
+    taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator),
+    apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -5731,6 +5734,7 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	state Span span(spanContext, location, { parent });
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - startTime;
+	cx->lastProxyRequest = startTime;
 	cx->updateCachedRV(startTime, rep.version);
 	// use startTime instead?
 	// maybe this also requires tracking number of loops processed in queue?

From e9c58d9f86f4670ebb8fafd1215794aeadc94ab7 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 19 Nov 2021 14:33:06 -0500
Subject: [PATCH 023/210] Check for sustained throttling in the proxy to lower
 threshold time and avoid false positives

---
 fdbclient/ClientKnobs.cpp          |  7 ++++---
 fdbclient/ClientKnobs.h            |  5 +++--
 fdbclient/CommitProxyInterface.h   |  6 ++++--
 fdbclient/NativeAPI.actor.cpp      |  9 +++------
 fdbserver/GrvProxyServer.actor.cpp | 27 +++++++++++++++++++++++----
 5 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index bb7940ce47..d0329c444b 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -115,9 +115,10 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( MAX_PROXY_CONTACT_LAG,                    1.0 );
-	init( DEBUG_USE_GRV_CACHE_CHANCE,              -0.1 ); // Since we check <= for 100% chance at 1.0, we don't want the default to be 0. This also means 0 is not 0%.
-	init( GRV_CACHE_RK_COOLDOWN,                    5.0 );
-	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,      0.01 );
+	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // Since we check <= for 100% chance at 1.0, we don't want the default to be 0. This also means 0 is not 0%.
+	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
+	init( GRV_THROTTLING_THRESHOLD,              0.0001 );
+	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index c48c9cc8e8..2c326f61fe 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -120,8 +120,9 @@ public:
 	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
-	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // Adjust what amount of time is considered "sustained" throttling on
-	                                           // proxy for a GRV which leads to disabling the cache
+	double GRV_THROTTLING_THRESHOLD; // Number of seconds spent in the request queue which is considered throttling
+	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
+	                                           // specified here, ratekeeper is throttling and not a false positive
 
 	// Taskbucket
 	double TASKBUCKET_LOGGING_DELAY;
diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index a3531da863..b0c975916d 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -196,10 +196,11 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
 	double timeThrottled;
+	bool rkThrottled;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
-	GetReadVersionReply() : version(invalidVersion), locked(false) {}
+	GetReadVersionReply() : version(invalidVersion), locked(false), rkThrottled(false) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -210,7 +211,8 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 		           metadataVersion,
 		           tagThrottleInfo,
 		           midShardSize,
-		           timeThrottled);
+		           timeThrottled,
+		           rkThrottled);
 	}
 };
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index ae8ad4fd9a..27fbeffe0f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5736,14 +5736,11 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	double latency = now() - startTime;
 	cx->lastProxyRequest = startTime;
 	cx->updateCachedRV(startTime, rep.version);
-	// use startTime instead?
-	// maybe this also requires tracking number of loops processed in queue?
 	TraceEvent("DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
-	if (rep.timeThrottled > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD &&
-	    priority != TransactionPriority::IMMEDIATE) {
+	if (rep.rkThrottled && priority != TransactionPriority::IMMEDIATE) {
 		TraceEvent("DebugGrvThrottleObserved")
-			.detail("TimeThrottled", rep.timeThrottled)
-			.detail("Threshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
+		    .detail("TimeThrottled", rep.timeThrottled)
+		    .detail("Threshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD);
 		cx->lastTimedRkThrottle = now();
 	}
 	cx->GRVLatencies.addSample(latency);
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index beab806a2b..6934722695 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -46,6 +46,9 @@ struct GrvProxyStats {
 	double percentageOfDefaultGRVQueueProcessed;
 	double percentageOfBatchGRVQueueProcessed;
 
+	bool lastTxnThrottled;
+	double throttleStartTime;
+
 	LatencySample defaultTxnGRVTimeInQueue;
 	LatencySample batchTxnGRVTimeInQueue;
 
@@ -98,10 +101,11 @@ struct GrvProxyStats {
 	    updatesFromRatekeeper("UpdatesFromRatekeeper", cc), leaseTimeouts("LeaseTimeouts", cc), systemGRVQueueSize(0),
 	    defaultGRVQueueSize(0), batchGRVQueueSize(0), transactionRateAllowed(0), batchTransactionRateAllowed(0),
 	    transactionLimit(0), batchTransactionLimit(0), percentageOfDefaultGRVQueueProcessed(0),
-	    percentageOfBatchGRVQueueProcessed(0), defaultTxnGRVTimeInQueue("DefaultTxnGRVTimeInQueue",
-	                                                                    id,
-	                                                                    SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
-	                                                                    SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+	    percentageOfBatchGRVQueueProcessed(0), lastTxnThrottled(false), throttleStartTime(0.0),
+	    defaultTxnGRVTimeInQueue("DefaultTxnGRVTimeInQueue",
+	                             id,
+	                             SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+	                             SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
 	    batchTxnGRVTimeInQueue("BatchTxnGRVTimeInQueue",
 	                           id,
 	                           SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
@@ -649,6 +653,21 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		}
 
 		reply.timeThrottled = request.reqProcessEnd - request.reqProcessStart;
+		if (reply.timeThrottled > CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
+			if (stats->lastTxnThrottled) {
+				// Check if this throttling has been sustained for a certain amount of time to avoid false positives
+				if (now() - stats->throttleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+					reply.rkThrottled = true;
+				}
+			} else { // !stats->lastTxnThrottled
+				// If not previously throttled, this request/reply is our new starting point
+				// for judging whether we are being actively throttled by ratekeeper now
+				stats->lastTxnThrottled = true;
+				stats->throttleStartTime = now();
+			}
+		} else { // reply.timeThrottled <= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD
+			stats->lastTxnThrottled = false;
+		}
 		request.reply.send(reply);
 		++stats->txnRequestOut;
 	}

From 33ee5fa3723da2fc1b2e242fdcdfaa4de6c68f08 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 22 Nov 2021 13:00:53 -0500
Subject: [PATCH 024/210] add tracing to proxy throttling check codepath

---
 fdbserver/GrvProxyServer.actor.cpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index f559cc6a21..3f1cdda786 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/TransactionLineage.h"
 #include "fdbserver/LogSystem.h"
@@ -653,19 +654,30 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		}
 
 		reply.timeThrottled = request.reqProcessEnd - request.reqProcessStart;
+		TraceEvent("DebugGrvProxyThrottleCheck")
+		    .detail("TimeThrottled", reply.timeThrottled)
+		    .detail("ThrottleThreshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD)
+		    .detail("LastTxnThrottled", stats->lastTxnThrottled)
+		    .detail("ThrottleStartTime", stats->throttleStartTime)
+		    .detail("SustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
 		if (reply.timeThrottled > CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
+			TraceEvent("DebugGrvProxyThrottled");
 			if (stats->lastTxnThrottled) {
+				TraceEvent("DebugGrvProxyLastTxnThrottled");
 				// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 				if (now() - stats->throttleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+					TraceEvent("DebugGrvProxyRkThrottled");
 					reply.rkThrottled = true;
 				}
 			} else { // !stats->lastTxnThrottled
+				TraceEvent("DebugGrvProxyLastTxnNotThrottled");
 				// If not previously throttled, this request/reply is our new starting point
 				// for judging whether we are being actively throttled by ratekeeper now
 				stats->lastTxnThrottled = true;
 				stats->throttleStartTime = now();
 			}
 		} else { // reply.timeThrottled <= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD
+			TraceEvent("DebugGrvProxyNotThrottled");
 			stats->lastTxnThrottled = false;
 		}
 		request.reply.send(reply);

From beae9ccfa1f77712083a6917b78deab1fe02992f Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 23 Nov 2021 14:58:17 -0500
Subject: [PATCH 025/210] tweak some knob default settings and trace formatting

---
 fdbclient/ClientKnobs.cpp          | 4 ++--
 fdbclient/ClientKnobs.h            | 2 ++
 fdbserver/GrvProxyServer.actor.cpp | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index e4b338e09e..15fc99a5d7 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -119,9 +119,9 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( MAX_PROXY_CONTACT_LAG,                    1.0 );
-	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // Since we check <= for 100% chance at 1.0, we don't want the default to be 0. This also means 0 is not 0%.
+	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0. 
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
-	init( GRV_THROTTLING_THRESHOLD,              0.0001 );
+	init( GRV_THROTTLING_THRESHOLD,              100e-9 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
 
 	// TaskBucket
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index e385f8030a..90cf578e78 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -123,6 +123,8 @@ public:
 	double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
 	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
+
+	// should these be server knobs?
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 	double GRV_THROTTLING_THRESHOLD; // Number of seconds spent in the request queue which is considered throttling
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 3f1cdda786..07bd1fdc57 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -658,7 +658,8 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		    .detail("TimeThrottled", reply.timeThrottled)
 		    .detail("ThrottleThreshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD)
 		    .detail("LastTxnThrottled", stats->lastTxnThrottled)
-		    .detail("ThrottleStartTime", stats->throttleStartTime)
+		    .detail("ThrottleStartTime", format("%.6f", stats->throttleStartTime))
+		    .detail("Diff", now() - stats->throttleStartTime)
 		    .detail("SustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
 		if (reply.timeThrottled > CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
 			TraceEvent("DebugGrvProxyThrottled");

From bba7ed8ac06d7bce47a96b5af9e40ffca12e4412 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 25 Nov 2021 13:22:28 -0500
Subject: [PATCH 026/210] Add panic knob to shut down cache from outside txn
 options

---
 fdbclient/ClientKnobs.cpp     | 1 +
 fdbclient/ClientKnobs.h       | 2 +-
 fdbclient/NativeAPI.actor.cpp | 9 +++++++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 15fc99a5d7..24981e333f 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -120,6 +120,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
 	init( MAX_PROXY_CONTACT_LAG,                    1.0 );
 	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0. 
+	init( FORCE_GRV_CACHE_OFF,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
 	init( GRV_THROTTLING_THRESHOLD,              100e-9 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 90cf578e78..8c65a10fe7 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -123,7 +123,7 @@ public:
 	double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
 	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
-
+	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
 	// should these be server knobs?
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 	double GRV_THROTTLING_THRESHOLD; // Number of seconds spent in the request queue which is considered throttling
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index e6e86edf4d..64a3afbf39 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -31,6 +31,7 @@
 #include "contrib/fmt-8.0.1/include/fmt/format.h"
 
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/Knobs.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/MultiInterface.h"
 
@@ -6033,8 +6034,9 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
-		if (!options.skipGrvCache && rkThrottlingCooledDown(getDatabase().getPtr()) &&
-		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache)) {
+		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
+		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
+		    rkThrottlingCooledDown(getDatabase().getPtr())) {
 			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", format("%.6f", cx->lastTimedGrv.get()));
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
@@ -6043,6 +6045,9 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
 			return readVersion;
 		}
+		if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && cx->grvUpdateHandler.isValid()) {
+			cx->grvUpdateHandler.cancel();
+		}
 		++cx->transactionReadVersions;
 		flags |= options.getReadVersionFlags;
 		switch (options.priority) {

From 2b0ade5250debf7e5ff0d954b5801e40144de968 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 25 Nov 2021 13:31:55 -0500
Subject: [PATCH 027/210] Change throttling threshold to count loop iterations
 instead of time

---
 fdbclient/ClientKnobs.cpp          |  2 +-
 fdbclient/ClientKnobs.h            |  2 +-
 fdbclient/CommitProxyInterface.h   | 12 ++++++------
 fdbclient/NativeAPI.actor.cpp      |  5 ++---
 fdbserver/GrvProxyServer.actor.cpp | 19 ++++---------------
 5 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 24981e333f..a87ae5c4fa 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -122,7 +122,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0. 
 	init( FORCE_GRV_CACHE_OFF,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
-	init( GRV_THROTTLING_THRESHOLD,              100e-9 );
+	init( GRV_THROTTLING_THRESHOLD,                   2 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
 
 	// TaskBucket
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 8c65a10fe7..b60c902197 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -126,7 +126,7 @@ public:
 	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
 	// should these be server knobs?
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
-	double GRV_THROTTLING_THRESHOLD; // Number of seconds spent in the request queue which is considered throttling
+	int GRV_THROTTLING_THRESHOLD; // Number of iterations spent in the request queue loop which is considered throttling
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
 	                                           // specified here, ratekeeper is throttling and not a false positive
 
diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index d67b17cd42..cc86294d6f 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -198,7 +198,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	bool locked;
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
-	double timeThrottled;
+	uint32_t queueIterations = 0;
 	bool rkThrottled;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
@@ -214,7 +214,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 		           metadataVersion,
 		           tagThrottleInfo,
 		           midShardSize,
-		           timeThrottled,
+		           queueIterations,
 		           rkThrottled);
 	}
 };
@@ -238,14 +238,14 @@ struct GetReadVersionRequest : TimedRequest {
 	uint32_t transactionCount;
 	uint32_t flags;
 	TransactionPriority priority;
-	double reqProcessStart, reqProcessEnd;
+	uint32_t queueIterations;
 
 	TransactionTagMap<uint32_t> tags;
 
 	Optional<UID> debugID;
 	ReplyPromise<GetReadVersionReply> reply;
 
-	GetReadVersionRequest() : transactionCount(1), flags(0), reqProcessStart(0.0), reqProcessEnd(0.0) {}
+	GetReadVersionRequest() : transactionCount(1), flags(0), queueIterations(0) {}
 	GetReadVersionRequest(SpanID spanContext,
 	                      uint32_t transactionCount,
 	                      TransactionPriority priority,
@@ -253,7 +253,7 @@ struct GetReadVersionRequest : TimedRequest {
 	                      TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
 	                      Optional<UID> debugID = Optional<UID>())
 	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority),
-	    reqProcessStart(0.0), reqProcessEnd(0.0), tags(tags), debugID(debugID) {
+	    queueIterations(0), tags(tags), debugID(debugID) {
 		flags = flags & ~FLAG_PRIORITY_MASK;
 		switch (priority) {
 		case TransactionPriority::BATCH:
@@ -274,7 +274,7 @@ struct GetReadVersionRequest : TimedRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, reqProcessStart, reqProcessEnd);
+		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, queueIterations);
 
 		if (ar.isDeserializing) {
 			if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 64a3afbf39..2df4caa810 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5333,7 +5333,6 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
-				TraceEvent("DebugGrvCommitSuccess");
 				cx->updateCachedRV(grvTime, v);
 				if (v != invalidVersion) {
 					if (CLIENT_BUGGIFY) {
@@ -5950,10 +5949,10 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	double latency = now() - startTime;
 	cx->lastProxyRequest = startTime;
 	cx->updateCachedRV(startTime, rep.version);
-	TraceEvent("DebugGrvTimeThrottled").detail("TimeThrottled", rep.timeThrottled);
+	TraceEvent("DebugGrvQueueIterations").detail("TimeThrottled", rep.queueIterations);
 	if (rep.rkThrottled && priority != TransactionPriority::IMMEDIATE) {
 		TraceEvent("DebugGrvThrottleObserved")
-		    .detail("TimeThrottled", rep.timeThrottled)
+		    .detail("TimeThrottled", rep.queueIterations)
 		    .detail("Threshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD);
 		cx->lastTimedRkThrottle = now();
 	}
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 07bd1fdc57..33c56cbb30 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -653,15 +653,15 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			}
 		}
 
-		reply.timeThrottled = request.reqProcessEnd - request.reqProcessStart;
+		reply.queueIterations = request.queueIterations;
 		TraceEvent("DebugGrvProxyThrottleCheck")
-		    .detail("TimeThrottled", reply.timeThrottled)
+		    .detail("QueueIterations", reply.queueIterations)
 		    .detail("ThrottleThreshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD)
 		    .detail("LastTxnThrottled", stats->lastTxnThrottled)
 		    .detail("ThrottleStartTime", format("%.6f", stats->throttleStartTime))
 		    .detail("Diff", now() - stats->throttleStartTime)
 		    .detail("SustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
-		if (reply.timeThrottled > CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
+		if (reply.queueIterations >= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
 			TraceEvent("DebugGrvProxyThrottled");
 			if (stats->lastTxnThrottled) {
 				TraceEvent("DebugGrvProxyLastTxnThrottled");
@@ -842,25 +842,16 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			// transactionQueue->span.swap(span);
 
 			auto& req = transactionQueue->front();
+			req.queueIterations++;
 			int tc = req.transactionCount;
 
 			if (req.priority < TransactionPriority::DEFAULT &&
 			    !batchRateInfo.canStart(transactionsStarted[0] + transactionsStarted[1], tc)) {
-				if (req.reqProcessStart == 0.0) {
-					req.reqProcessStart = g_network->timer();
-				}
 				break;
 			} else if (req.priority < TransactionPriority::IMMEDIATE &&
 			           !normalRateInfo.canStart(transactionsStarted[0] + transactionsStarted[1], tc)) {
-				if (req.reqProcessStart == 0.0) {
-					req.reqProcessStart = g_network->timer();
-				}
 				break;
 			}
-			// If not previously set, this is the first time this transaction is being processed.
-			if (req.reqProcessStart == 0.0) {
-				req.reqProcessStart = g_network->timer();
-			}
 
 			if (req.debugID.present()) {
 				if (!debugID.present())
@@ -882,8 +873,6 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 				grvProxyData->stats.batchTxnGRVTimeInQueue.addMeasurement(currentTime - req.requestTime());
 				--grvProxyData->stats.batchGRVQueueSize;
 			}
-
-			req.reqProcessEnd = g_network->timer();
 			start[req.flags & 1].push_back(std::move(req));
 			static_assert(GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY == 1, "Implementation dependent on flag value");
 			transactionQueue->pop_front();

From 1fb537cde5fc0951016233c301b0f7cf1b30b994 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 29 Nov 2021 14:16:49 -0500
Subject: [PATCH 028/210] Add cache generations to create time boundaries when
 commits time out or fail

---
 fdbclient/DatabaseContext.h   |  2 ++
 fdbclient/NativeAPI.actor.cpp | 21 +++++++++++++++++----
 fdbclient/NativeAPI.actor.h   |  2 ++
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index fd9aa37f4a..cc5d336ac1 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -435,11 +435,13 @@ public:
 	NotifiedDouble lastTimedGrv;
 	Version cachedRv;
 	void updateCachedRV(double t, Version v);
+	void invalidateCache();
 	double lastTimedRkThrottle;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
 	// we want to track the last time in order to periodically contact the proxy to check for throttling
 	double lastProxyRequest;
+	Version rvCacheGeneration;
 
 	int snapshotRywEnabled;
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 2df4caa810..49470c341c 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -214,6 +214,11 @@ void DatabaseContext::updateCachedRV(double t, Version v) {
 	}
 }
 
+void DatabaseContext::invalidateCache() {
+	cachedRv = 0;
+	lastTimedGrv = 0.0;
+}
+
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
                                                              StorageServerInterface const& ssi,
                                                              LocalityData const& locality) {
@@ -1291,9 +1296,9 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
     bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0),
-    lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
-    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    lastProxyRequest(0.0), rvCacheGeneration(0), transactionTracingSample(false), taskID(taskID),
+    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
+    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -5328,12 +5333,17 @@ ACTOR static Future<Void> tryCommit(Database cx,
 		state double grvTime = now();
 		choose {
 			when(wait(cx->onProxiesChanged())) {
+				cx->rvCacheGeneration++;
 				reply.cancel();
 				throw request_maybe_delivered();
 			}
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
-				cx->updateCachedRV(grvTime, v);
+				if (cx->rvCacheGeneration != tr->getRvGeneration()) {
+					cx->invalidateCache();
+				} else {
+					cx->updateCachedRV(grvTime, v);
+				}
 				if (v != invalidVersion) {
 					if (CLIENT_BUGGIFY) {
 						throw commit_unknown_result();
@@ -5372,6 +5382,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 					return Void();
 				} else {
 					// clear the RYW transaction which contains previous conflicting keys
+					cx->rvCacheGeneration++;
 					tr->info.conflictingKeys.reset();
 					if (ci.conflictingKRIndices.present()) {
 						tr->info.conflictingKeys =
@@ -5400,6 +5411,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 		}
 	} catch (Error& e) {
+		cx->rvCacheGeneration++;
 		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result) {
 			// We don't know if the commit happened, and it might even still be in flight.
 
@@ -6042,6 +6054,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
 			}
 			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
+			rvGeneration = cx->rvCacheGeneration;
 			return readVersion;
 		}
 		if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && cx->grvUpdateHandler.isValid()) {
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index a36ceb4c57..7c2a799631 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -430,6 +430,7 @@ public:
 
 	void setTransactionID(uint64_t id);
 	void setToken(uint64_t token);
+	Version getRvGeneration() { return rvGeneration; }
 
 	const std::vector<Future<std::pair<Key, Key>>>& getExtraReadConflictRanges() const { return extraConflictRanges; }
 	Standalone<VectorRef<KeyRangeRef>> readConflictRanges() const {
@@ -445,6 +446,7 @@ private:
 
 	double backoff;
 	Version committedVersion{ invalidVersion };
+	Version rvGeneration;
 	CommitTransactionRequest tr;
 	Future<Version> readVersion;
 	Promise<Optional<Value>> metadataVersion;

From b5b0afb96d7697cb3fc70aff10cba61ce316595e Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 29 Nov 2021 14:38:52 -0500
Subject: [PATCH 029/210] narrow down places where to advance cache generation

---
 fdbclient/NativeAPI.actor.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 49470c341c..d2850f3451 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5333,7 +5333,6 @@ ACTOR static Future<Void> tryCommit(Database cx,
 		state double grvTime = now();
 		choose {
 			when(wait(cx->onProxiesChanged())) {
-				cx->rvCacheGeneration++;
 				reply.cancel();
 				throw request_maybe_delivered();
 			}
@@ -5382,7 +5381,6 @@ ACTOR static Future<Void> tryCommit(Database cx,
 					return Void();
 				} else {
 					// clear the RYW transaction which contains previous conflicting keys
-					cx->rvCacheGeneration++;
 					tr->info.conflictingKeys.reset();
 					if (ci.conflictingKRIndices.present()) {
 						tr->info.conflictingKeys =
@@ -5411,10 +5409,11 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 		}
 	} catch (Error& e) {
-		cx->rvCacheGeneration++;
 		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result) {
 			// We don't know if the commit happened, and it might even still be in flight.
 
+			// Advance the cached RV generation to create a time boundary where a commit (possibly) failed.
+			cx->rvCacheGeneration++;
 			if (!options.causalWriteRisky) {
 				// Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the
 				// version we submitted with is dead, or by committing a conflicting transaction successfully

From dcb0fc82e9ede5ea95ea9403d4d1124d92e61ec5 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 29 Nov 2021 15:13:48 -0500
Subject: [PATCH 030/210] remove future wait optimization with NotifiedDouble
 and use regular GRV path if cache is too stale

---
 fdbclient/DatabaseContext.h   |  4 ++--
 fdbclient/NativeAPI.actor.cpp | 44 +++++++++++++++++++----------------
 fdbclient/NativeAPI.actor.h   |  2 +-
 3 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index cc5d336ac1..ee4048a721 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -432,10 +432,10 @@ public:
 
 	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
-	NotifiedDouble lastTimedGrv;
+	double lastTimedGrv;
 	Version cachedRv;
 	void updateCachedRV(double t, Version v);
-	void invalidateCache();
+	void invalidateRvCache();
 	double lastTimedRkThrottle;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index d2850f3451..891f5c1076 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -203,18 +203,18 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 }
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (t > lastTimedGrv.get() && v >= cachedRv) {
+	if (t > lastTimedGrv && v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")
 		    .detail("Version", v)
 		    .detail("CurTime", t)
 		    .detail("LastVersion", cachedRv)
-		    .detail("LastTime", lastTimedGrv.get());
+		    .detail("LastTime", lastTimedGrv);
 		cachedRv = v;
 		lastTimedGrv = t;
 	}
 }
 
-void DatabaseContext::invalidateCache() {
+void DatabaseContext::invalidateRvCache() {
 	cachedRv = 0;
 	lastTimedGrv = 0.0;
 }
@@ -1032,14 +1032,14 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 		loop {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
-			state double lastTime = cx->lastTimedGrv.get();
+			state double lastTime = cx->lastTimedGrv;
 			state double lastProxyTime = cx->lastProxyRequest;
 			TraceEvent("BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
 			    .detail("GrvDelay", grvDelay)
 			    .detail("CachedRv", cx->cachedRv)
-			    .detail("CachedTime", cx->lastTimedGrv.get())
+			    .detail("CachedTime", cx->lastTimedGrv)
 			    .detail("Gap", curTime - lastTime)
 			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
 			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) ||
@@ -1052,7 +1052,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					TraceEvent("BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
 					    .detail("CachedRv", cx->cachedRv)
-					    .detail("CachedTime", cx->lastTimedGrv.get());
+					    .detail("CachedTime", cx->lastTimedGrv);
 				} catch (Error& e) {
 					TraceEvent("BackgroundGrvUpdaterTxnError").error(e, true);
 					wait(tr.onError(e));
@@ -5339,7 +5339,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
 				if (cx->rvCacheGeneration != tr->getRvGeneration()) {
-					cx->invalidateCache();
+					cx->invalidateRvCache();
 				} else {
 					cx->updateCachedRV(grvTime, v);
 				}
@@ -6021,15 +6021,15 @@ ACTOR Future<Version> extractReadVersion(Location location,
 	return rep.version;
 }
 
-ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double requestTime) {
-	if (requestTime - cx->lastTimedGrv.get() > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
-		wait(cx->lastTimedGrv.whenAtLeast(requestTime - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
-	}
-	// Want to check that the cached version is at most
-	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
-	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
-	return cx->cachedRv;
-}
+// ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double requestTime) {
+// 	if (requestTime - cx->lastTimedGrv.get() > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+// 		wait(cx->lastTimedGrv.whenAtLeast(requestTime - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
+// 	}
+// 	// Want to check that the cached version is at most
+// 	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
+// 	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
+// 	return cx->cachedRv;
+// }
 
 bool rkThrottlingCooledDown(DatabaseContext* cx) {
 	if (cx->lastTimedRkThrottle == 0.0) {
@@ -6043,18 +6043,22 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 }
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
+	rvGeneration = cx->rvCacheGeneration;
 	if (!readVersion.isValid()) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
 		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
 		    rkThrottlingCooledDown(getDatabase().getPtr())) {
-			TraceEvent("DebugGrvUseCache").detail("LastRV", cx->cachedRv).detail("LastTime", format("%.6f", cx->lastTimedGrv.get()));
+			TraceEvent("DebugGrvUseCache")
+			    .detail("LastRV", cx->cachedRv)
+			    .detail("LastTime", format("%.6f", cx->lastTimedGrv));
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
 			}
-			readVersion = getDBCachedReadVersion(getDatabase().getPtr(), now());
-			rvGeneration = cx->rvCacheGeneration;
-			return readVersion;
+			if (now() - cx->lastTimedGrv <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+				readVersion = cx->cachedRv;
+				return readVersion;
+			} // else go through regular GRV path
 		}
 		if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && cx->grvUpdateHandler.isValid()) {
 			cx->grvUpdateHandler.cancel();
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index 7c2a799631..dc9797134c 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -446,7 +446,7 @@ private:
 
 	double backoff;
 	Version committedVersion{ invalidVersion };
-	Version rvGeneration;
+	Version rvGeneration{ invalidVersion };
 	CommitTransactionRequest tr;
 	Future<Version> readVersion;
 	Promise<Optional<Value>> metadataVersion;

From 00ffd99938804cee8e39ca505f2b276d6cb3c104 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 30 Nov 2021 16:03:19 -0500
Subject: [PATCH 031/210] Clone sideband workload to check consistency of cache

---
 fdbserver/CMakeLists.txt                     |   1 +
 fdbserver/workloads/SidebandSingle.actor.cpp | 210 +++++++++++++++++++
 tests/CMakeLists.txt                         |   1 +
 tests/fast/SidebandSingle.toml               |  30 +++
 4 files changed, 242 insertions(+)
 create mode 100644 fdbserver/workloads/SidebandSingle.actor.cpp
 create mode 100644 tests/fast/SidebandSingle.toml

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index c1ed1d01ae..20301e4058 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -236,6 +236,7 @@ set(FDBSERVER_SRCS
   workloads/SelectorCorrectness.actor.cpp
   workloads/Serializability.actor.cpp
   workloads/Sideband.actor.cpp
+  workloads/SidebandSingle.actor.cpp
   workloads/SimpleAtomicAdd.actor.cpp
   workloads/SlowTaskWorkload.actor.cpp
   workloads/SnapTest.actor.cpp
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
new file mode 100644
index 0000000000..a35024212d
--- /dev/null
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -0,0 +1,210 @@
+/*
+ * SidebandSingle.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/Knobs.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct SidebandMessage {
+	constexpr static FileIdentifier file_identifier = 11862047;
+	uint64_t key;
+	Version commitVersion;
+
+	SidebandMessage() {}
+	SidebandMessage(uint64_t key, Version commitVersion) : key(key), commitVersion(commitVersion) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, key, commitVersion);
+	}
+};
+
+struct SidebandInterface {
+	constexpr static FileIdentifier file_identifier = 15950545;
+	RequestStream<SidebandMessage> updates;
+
+	UID id() const { return updates.getEndpoint().token; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, updates);
+	}
+};
+
+struct SidebandSingleWorkload : TestWorkload {
+	double testDuration, operationsPerSecond;
+	SidebandInterface interf;
+
+	std::vector<Future<Void>> clients;
+	PerfIntCounter messages, consistencyErrors, keysUnexpectedlyPresent;
+
+	SidebandSingleWorkload(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), messages("Messages"), consistencyErrors("Causal Consistency Errors"),
+	    keysUnexpectedlyPresent("KeysUnexpectedlyPresent") {
+		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		operationsPerSecond = getOption(options, LiteralStringRef("operationsPerSecond"), 50.0);
+	}
+
+	std::string description() const override { return "SidebandSingleWorkload"; }
+	Future<Void> setup(Database const& cx) override {
+		if (clientId != 0)
+			return Void();
+		return persistInterface(this, cx);
+	}
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0)
+			return Void();
+
+		clients.push_back(mutator(this, cx));
+		clients.push_back(checker(this, cx));
+		return delay(testDuration);
+	}
+	Future<bool> check(Database const& cx) override {
+		if (clientId != 0)
+			return true;
+		int errors = 0;
+		for (int c = 0; c < clients.size(); c++) {
+			errors += clients[c].isError();
+		}
+		if (errors)
+			TraceEvent(SevError, "TestFailure").detail("Reason", "There were client errors.");
+		clients.clear();
+		if (consistencyErrors.getValue())
+			TraceEvent(SevError, "TestFailure").detail("Reason", "There were causal consistency errors.");
+		return !errors && !consistencyErrors.getValue();
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {
+		m.push_back(messages.getMetric());
+		m.push_back(consistencyErrors.getMetric());
+		m.push_back(keysUnexpectedlyPresent.getMetric());
+	}
+
+	ACTOR Future<Void> persistInterface(SidebandSingleWorkload* self, Database cx) {
+		state Transaction tr(cx);
+		BinaryWriter wr(IncludeVersion());
+		wr << self->interf;
+		state Standalone<StringRef> serializedInterface = wr.toValue();
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
+				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
+				if (val.present()) {
+					if (val.get() != serializedInterface)
+						throw operation_failed();
+					break;
+				}
+				tr.set(format("Sideband/Client/%d", self->clientId), serializedInterface);
+				wait(tr.commit());
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		TraceEvent("SidebandPersisted", self->interf.id()).detail("ClientIdx", self->clientId);
+		return Void();
+	}
+
+	ACTOR Future<SidebandInterface> fetchSideband(SidebandSingleWorkload* self, Database cx) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
+				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
+				if (!val.present()) {
+					throw operation_failed();
+				}
+				SidebandInterface sideband;
+				BinaryReader br(val.get(), IncludeVersion());
+				br >> sideband;
+				TraceEvent("SidebandFetched", sideband.id()).detail("ClientIdx", self->clientId);
+				return sideband;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR Future<Void> mutator(SidebandSingleWorkload* self, Database cx) {
+		state SidebandInterface checker = wait(self->fetchSideband(self, cx));
+		state double lastTime = now();
+		state Version commitVersion;
+
+		loop {
+			wait(poisson(&lastTime, 1.0 / self->operationsPerSecond));
+			state Transaction tr(cx);
+			state uint64_t key = deterministicRandom()->randomUniqueID().hash();
+
+			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
+					Optional<Value> val = wait(tr.get(messageKey));
+					if (val.present()) {
+						commitVersion = tr.getReadVersion().get();
+						++self->keysUnexpectedlyPresent;
+						break;
+					}
+					tr.set(messageKey, LiteralStringRef("deadbeef"));
+					wait(tr.commit());
+					commitVersion = tr.getCommittedVersion();
+					break;
+				} catch (Error& e) {
+					state Error error = e;
+					wait(tr.onError(e));
+					if (error.code() == error_code_commit_unknown_result) {
+						// something
+					}
+				}
+			}
+			++self->messages;
+			checker.updates.send(SidebandMessage(key, commitVersion));
+		}
+	}
+
+	ACTOR Future<Void> checker(SidebandSingleWorkload* self, Database cx) {
+		loop {
+			state SidebandMessage message = waitNext(self->interf.updates.getFuture());
+			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.key));
+			state Transaction tr(cx);
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
+					Optional<Value> val = wait(tr.get(messageKey));
+					if (!val.present()) {
+						TraceEvent(SevError, "CausalConsistencyError", self->interf.id())
+						    .detail("MessageKey", messageKey.toString().c_str())
+						    .detail("RemoteCommitVersion", message.commitVersion)
+						    .detail("LocalReadVersion",
+						            tr.getReadVersion().get()); // will assert that ReadVersion is set
+						++self->consistencyErrors;
+					}
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+};
+
+WorkloadFactory<SidebandSingleWorkload> SidebandSingleWorkloadFactory("SidebandSingle");
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index a7840c6a28..508be73812 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -158,6 +158,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/ReportConflictingKeys.toml)
   add_fdb_test(TEST_FILES fast/SelectorCorrectness.toml)
   add_fdb_test(TEST_FILES fast/Sideband.toml)
+  add_fdb_test(TEST_FILES fast/SidebandSingle.toml)
   add_fdb_test(TEST_FILES fast/SidebandWithStatus.toml)
   add_fdb_test(TEST_FILES fast/SimpleAtomicAdd.toml)
   add_fdb_test(TEST_FILES fast/SpecialKeySpaceCorrectness.toml)
diff --git a/tests/fast/SidebandSingle.toml b/tests/fast/SidebandSingle.toml
new file mode 100644
index 0000000000..b244431c85
--- /dev/null
+++ b/tests/fast/SidebandSingle.toml
@@ -0,0 +1,30 @@
+[[test]]
+testTitle = 'SingleClientCausalConsistencyTest'
+
+    [[test.workload]]
+    testName = 'SidebandSingle'
+    testDuration = 30.0
+    operationsPerSecond = 500
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 10.0
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0

From 7fab9ea5c0fee44133c226992ad14584b268ae66 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 10 Dec 2021 16:51:12 -0500
Subject: [PATCH 032/210] adjust sideband workload to expose another
 consistency error

---
 fdbclient/NativeAPI.actor.cpp                | 35 ++++++---
 fdbserver/workloads/SidebandSingle.actor.cpp | 81 +++++++++++++++-----
 2 files changed, 86 insertions(+), 30 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 891f5c1076..58618edff2 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -203,14 +203,20 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 }
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (t > lastTimedGrv && v >= cachedRv) {
+	if (v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")
 		    .detail("Version", v)
 		    .detail("CurTime", t)
 		    .detail("LastVersion", cachedRv)
 		    .detail("LastTime", lastTimedGrv);
 		cachedRv = v;
-		lastTimedGrv = t;
+		// Since the time is based on the start of the request, it's possible that we
+		// get a newer version with an older time.
+		// (Request started earlier, but was latest to reach the proxy)
+		// Only update time when strictly increasing (?)
+		if (t > lastTimedGrv) {
+			lastTimedGrv = t;
+		}
 	}
 }
 
@@ -5338,15 +5344,18 @@ ACTOR static Future<Void> tryCommit(Database cx,
 			}
 			when(CommitID ci = wait(reply)) {
 				Version v = ci.version;
-				if (cx->rvCacheGeneration != tr->getRvGeneration()) {
-					cx->invalidateRvCache();
-				} else {
-					cx->updateCachedRV(grvTime, v);
-				}
 				if (v != invalidVersion) {
 					if (CLIENT_BUGGIFY) {
 						throw commit_unknown_result();
 					}
+					// TraceEvent("CheckpointSideband1");
+					// if (cx->rvCacheGeneration != tr->getRvGeneration()) {
+					// 	TraceEvent("CheckpointSideband1.1");
+					// 	cx->invalidateRvCache();
+					// } else {
+					TraceEvent("CheckpointSideband1.2");
+					cx->updateCachedRV(grvTime, v);
+					// }
 					if (info.debugID.present())
 						TraceEvent(interval.end()).detail("CommittedVersion", v);
 					*pCommittedVersion = v;
@@ -5411,6 +5420,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 	} catch (Error& e) {
 		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result) {
 			// We don't know if the commit happened, and it might even still be in flight.
+			TraceEvent("DebugSidebandCommitUnknownResult");
 
 			// Advance the cached RV generation to create a time boundary where a commit (possibly) failed.
 			cx->rvCacheGeneration++;
@@ -6048,14 +6058,15 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
 		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
 		    rkThrottlingCooledDown(getDatabase().getPtr())) {
-			TraceEvent("DebugGrvUseCache")
-			    .detail("LastRV", cx->cachedRv)
-			    .detail("LastTime", format("%.6f", cx->lastTimedGrv));
+			TraceEvent("DebugGrvEnterCachePath");
 			// Upon our first request to use cached RVs, start the background updater
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
 			}
-			if (now() - cx->lastTimedGrv <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
+			if (now() - cx->lastTimedGrv <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && cx->cachedRv != Version(0)) {
+				TraceEvent("DebugGrvUseCache")
+				    .detail("LastRV", cx->cachedRv)
+				    .detail("LastTime", format("%.6f", cx->lastTimedGrv));
 				readVersion = cx->cachedRv;
 				return readVersion;
 			} // else go through regular GRV path
@@ -6137,6 +6148,8 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		                                 startTime,
 		                                 metadataVersion,
 		                                 options.tags);
+	} else {
+		TraceEvent("DebugGrvValidRv");
 	}
 	return readVersion;
 }
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index a35024212d..66cfc43180 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -106,7 +106,6 @@ struct SidebandSingleWorkload : TestWorkload {
 		state Standalone<StringRef> serializedInterface = wr.toValue();
 		loop {
 			try {
-				tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
 				if (val.present()) {
 					if (val.get() != serializedInterface)
@@ -128,7 +127,6 @@ struct SidebandSingleWorkload : TestWorkload {
 		state Transaction tr(cx);
 		loop {
 			try {
-				tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
 				if (!val.present()) {
 					throw operation_failed();
@@ -148,34 +146,50 @@ struct SidebandSingleWorkload : TestWorkload {
 		state SidebandInterface checker = wait(self->fetchSideband(self, cx));
 		state double lastTime = now();
 		state Version commitVersion;
+		state bool unknown = false;
 
 		loop {
 			wait(poisson(&lastTime, 1.0 / self->operationsPerSecond));
+			state Transaction tr0(cx);
 			state Transaction tr(cx);
 			state uint64_t key = deterministicRandom()->randomUniqueID().hash();
 
 			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
+			// first set, this is the "old" value, always retry
 			loop {
 				try {
-					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
-					Optional<Value> val = wait(tr.get(messageKey));
-					if (val.present()) {
-						commitVersion = tr.getReadVersion().get();
-						++self->keysUnexpectedlyPresent;
-						break;
-					}
-					tr.set(messageKey, LiteralStringRef("deadbeef"));
-					wait(tr.commit());
-					commitVersion = tr.getCommittedVersion();
+					tr0.set(messageKey, LiteralStringRef("oldbeef"));
+					wait(tr0.commit());
 					break;
 				} catch (Error& e) {
-					state Error error = e;
-					wait(tr.onError(e));
-					if (error.code() == error_code_commit_unknown_result) {
-						// something
-					}
+					wait(tr0.onError(e));
 				}
 			}
+			// second set, the checker should see this, no retries on unknown result
+			loop {
+				try {
+					// tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
+					tr.set(messageKey, LiteralStringRef("deadbeef"));
+					TraceEvent("DebugSidebandBeforeCommit");
+					wait(tr.commit());
+					commitVersion = tr.getCommittedVersion();
+					TraceEvent("DebugSidebandAfterCommit").detail("CommitVersion", commitVersion);
+					break;
+				} catch (Error& e) {
+					if (e.code() == error_code_commit_unknown_result) {
+						TraceEvent("DebugSidebandUnknownResult");
+						unknown = true;
+						++self->messages;
+						checker.updates.send(SidebandMessage(key, invalidVersion));
+						break;
+					}
+					wait(tr.onError(e));
+				}
+			}
+			if (unknown) {
+				unknown = false;
+				continue;
+			}
 			++self->messages;
 			checker.updates.send(SidebandMessage(key, commitVersion));
 		}
@@ -188,18 +202,47 @@ struct SidebandSingleWorkload : TestWorkload {
 			state Transaction tr(cx);
 			loop {
 				try {
+					TraceEvent("DebugSidebandCacheGetBefore");
 					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
-					Optional<Value> val = wait(tr.get(messageKey));
+					state Optional<Value> val = wait(tr.get(messageKey));
+					TraceEvent("DebugSidebandCacheGetAfter");
 					if (!val.present()) {
-						TraceEvent(SevError, "CausalConsistencyError", self->interf.id())
+						TraceEvent(SevError, "CausalConsistencyError1", self->interf.id())
 						    .detail("MessageKey", messageKey.toString().c_str())
 						    .detail("RemoteCommitVersion", message.commitVersion)
 						    .detail("LocalReadVersion",
 						            tr.getReadVersion().get()); // will assert that ReadVersion is set
 						++self->consistencyErrors;
+					} else if (val.get() != LiteralStringRef("deadbeef")) {
+						TraceEvent("DebugSidebandOldBeef");
+						// check again without cache, and if it's the same, that's expected
+						state Transaction tr2(cx);
+						state Optional<Value> val2;
+						loop {
+							try {
+								TraceEvent("DebugSidebandNoCacheGetBefore");
+								Optional<Value> val2 = wait(tr2.get(messageKey));
+								TraceEvent("DebugSidebandNoCacheGetAfter");
+								break;
+							} catch (Error& e) {
+								TraceEvent("DebugSidebandNoCacheError").error(e, true);
+								wait(tr2.onError(e));
+							}
+						}
+						if (val != val2) {
+							TraceEvent(SevError, "CausalConsistencyError2", self->interf.id())
+							    .detail("MessageKey", messageKey.toString().c_str())
+							    .detail("Val1", val)
+							    .detail("Val2", val2)
+							    .detail("RemoteCommitVersion", message.commitVersion)
+							    .detail("LocalReadVersion",
+							            tr.getReadVersion().get()); // will assert that ReadVersion is set
+							++self->consistencyErrors;
+						}
 					}
 					break;
 				} catch (Error& e) {
+					TraceEvent("DebugSidebandCheckError").error(e, true);
 					wait(tr.onError(e));
 				}
 			}

From b7ca2a656cf41ed3e81c59583ebd720a10cd1ead Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 11 Jan 2022 13:35:43 -0500
Subject: [PATCH 033/210] remove epoch/generations for grv cache

---
 fdbclient/DatabaseContext.h                  |  2 --
 fdbclient/NativeAPI.actor.cpp                | 23 +++-----------------
 fdbserver/workloads/SidebandSingle.actor.cpp |  2 +-
 3 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index ee4048a721..0d32915183 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -435,13 +435,11 @@ public:
 	double lastTimedGrv;
 	Version cachedRv;
 	void updateCachedRV(double t, Version v);
-	void invalidateRvCache();
 	double lastTimedRkThrottle;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
 	// we want to track the last time in order to periodically contact the proxy to check for throttling
 	double lastProxyRequest;
-	Version rvCacheGeneration;
 
 	int snapshotRywEnabled;
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 58618edff2..800355869b 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -220,11 +220,6 @@ void DatabaseContext::updateCachedRV(double t, Version v) {
 	}
 }
 
-void DatabaseContext::invalidateRvCache() {
-	cachedRv = 0;
-	lastTimedGrv = 0.0;
-}
-
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
                                                              StorageServerInterface const& ssi,
                                                              LocalityData const& locality) {
@@ -1302,9 +1297,9 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
     bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0),
-    lastProxyRequest(0.0), rvCacheGeneration(0), transactionTracingSample(false), taskID(taskID),
-    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
-    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
+    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
+    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -5348,14 +5343,7 @@ ACTOR static Future<Void> tryCommit(Database cx,
 					if (CLIENT_BUGGIFY) {
 						throw commit_unknown_result();
 					}
-					// TraceEvent("CheckpointSideband1");
-					// if (cx->rvCacheGeneration != tr->getRvGeneration()) {
-					// 	TraceEvent("CheckpointSideband1.1");
-					// 	cx->invalidateRvCache();
-					// } else {
-					TraceEvent("CheckpointSideband1.2");
 					cx->updateCachedRV(grvTime, v);
-					// }
 					if (info.debugID.present())
 						TraceEvent(interval.end()).detail("CommittedVersion", v);
 					*pCommittedVersion = v;
@@ -5420,10 +5408,6 @@ ACTOR static Future<Void> tryCommit(Database cx,
 	} catch (Error& e) {
 		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result) {
 			// We don't know if the commit happened, and it might even still be in flight.
-			TraceEvent("DebugSidebandCommitUnknownResult");
-
-			// Advance the cached RV generation to create a time boundary where a commit (possibly) failed.
-			cx->rvCacheGeneration++;
 			if (!options.causalWriteRisky) {
 				// Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the
 				// version we submitted with is dead, or by committing a conflicting transaction successfully
@@ -6053,7 +6037,6 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 }
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
-	rvGeneration = cx->rvCacheGeneration;
 	if (!readVersion.isValid()) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
 		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index 66cfc43180..4c4c40ad9e 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -221,7 +221,7 @@ struct SidebandSingleWorkload : TestWorkload {
 						loop {
 							try {
 								TraceEvent("DebugSidebandNoCacheGetBefore");
-								Optional<Value> val2 = wait(tr2.get(messageKey));
+								wait(store(val2, tr2.get(messageKey)));
 								TraceEvent("DebugSidebandNoCacheGetAfter");
 								break;
 							} catch (Error& e) {

From da76abbdf65bf120da76f90a1e5ae5bbaecec449 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 11 Jan 2022 17:06:25 -0500
Subject: [PATCH 034/210] expose cluster ID through NativeAPI and C API

---
 bindings/c/fdb_c.cpp                        |  4 ++++
 bindings/c/foundationdb/fdb_c.h             |  2 ++
 fdbclient/DatabaseContext.h                 |  2 ++
 fdbclient/IClientApi.h                      |  2 ++
 fdbclient/MultiVersionTransaction.actor.cpp | 26 +++++++++++++++++++++
 fdbclient/MultiVersionTransaction.h         |  3 +++
 fdbclient/NativeAPI.actor.cpp               | 20 ++++++++++++++++
 fdbclient/ThreadSafeTransaction.cpp         |  5 ++++
 fdbclient/ThreadSafeTransaction.h           |  1 +
 9 files changed, 65 insertions(+)

diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index e5545251f2..4330b7cc91 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -358,6 +358,10 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
 	                        .extractPtr());
 }
 
+extern "C" DLLEXPORT FDBFuture* fdb_database_get_cluster_id(FDBDatabase* db) {
+	return (FDBFuture*)(DB(db)->getClusterId().extractPtr());
+}
+
 // Get network thread busyness (updated every 1s)
 // A value of 0 indicates that the client is more or less idle
 // A value of 1 (or more) indicates that the client is saturated
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index d3c65537c5..c85fbf6c25 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -187,6 +187,8 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_create_snapshot(FDBDatabase
                                                                      uint8_t const* snap_command,
                                                                      int snap_command_length);
 
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_cluster_id(FDBDatabase* db);
+
 DLLEXPORT WARN_UNUSED_RESULT double fdb_database_get_main_thread_busyness(FDBDatabase* db);
 
 DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db, uint64_t expected_version);
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 0d32915183..e83b63f260 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -274,6 +274,8 @@ public:
 	// Management API, create snapshot
 	Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
 
+	Future<UID> getClusterId();
+
 	Future<Void> getChangeFeedStream(Reference<ChangeFeedData> results,
 	                                 Key rangeID,
 	                                 Version begin = 0,
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index 5562ad3ba6..86e96c4e3a 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -130,6 +130,8 @@ public:
 	// Management API, create snapshot
 	virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
 
+	virtual ThreadFuture<UID> getClusterId() = 0;
+
 	// used in template functions as the Transaction type that can be created through createTransaction()
 	using TransactionT = ITransaction;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index fb2fc184ea..0bf8a091c6 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -398,6 +398,25 @@ double DLDatabase::getMainThreadBusyness() {
 	return 0;
 }
 
+ThreadFuture<UID> DLDatabase::getClusterId() {
+	if (api->getClusterId != nullptr) {
+		FdbCApi::FDBFuture* f = api->getClusterId();
+		return toThreadFuture<UID>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+			const uint8_t* res;
+			int resLen;
+			FdbCApi::fdb_error_t error = api->futureGetKey(f, &res, &resLen);
+			ASSERT(!error);
+			// UID loading code taken from
+			// IRandom.h:load(const uint8_t* i, UID& out, Context& context)
+			const uint64_t* in = reinterpret_cast<const uint64_t*>(res);
+			UID out = UID(in[0], in[1]);
+			return out;
+		});
+	}
+
+	return UID();
+}
+
 // Returns the protocol version reported by the coordinator this client is connected to
 // If an expected version is given, the future won't return until the protocol version is different than expected
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
@@ -1183,6 +1202,13 @@ double MultiVersionDatabase::getMainThreadBusyness() {
 	return localClientBusyness;
 }
 
+ThreadFuture<UID> MultiVersionDatabase::getClusterId() {
+	if (dbState->db) {
+		return dbState->db->getClusterId();
+	}
+	return UID();
+}
+
 // Returns the protocol version reported by the coordinator this client is connected to
 // If an expected version is given, the future won't return until the protocol version is different than expected
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 50e21bca57..10e4809200 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -84,6 +84,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                     int snapshotCommandLength);
 	double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
 	FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
+	FDBFuture* (*getClusterId)();
 
 	// Transaction
 	fdb_error_t (*transactionSetOption)(FDBTransaction* tr,
@@ -307,6 +308,7 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
+	ThreadFuture<UID> getClusterId() override;
 
 private:
 	const Reference<FdbCApi> api;
@@ -524,6 +526,7 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
+	ThreadFuture<UID> getClusterId() override;
 
 	// private:
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 800355869b..37193adcd5 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6910,6 +6910,26 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
 	return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
 }
 
+ACTOR static Future<UID> getClusterIdActor(DatabaseContext* cx) {
+	Database db(cx);
+	state Transaction tr(db);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> clusterId = wait(tr.get(clusterIdKey));
+			ASSERT(clusterId.present());
+			return BinaryReader::fromStringRef<UID>(clusterId.get(), Unversioned());
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+Future<UID> DatabaseContext::getClusterId() {
+	return getClusterIdActor(this);
+}
+
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
 	loop {
 		if (self->version.get() < self->desired.get()) {
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index 88ac4b6919..c14f768be4 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -95,6 +95,11 @@ ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons
 	return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
 }
 
+ThreadFuture<UID> ThreadSafeDatabase::getClusterId() {
+	DatabaseContext* db = this->db;
+	return onMainThread([db]() -> Future<UID> { return db->getClusterId(); });
+}
+
 // Return the main network thread busyness
 double ThreadSafeDatabase::getMainThreadBusyness() {
 	ASSERT(g_network);
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index 1f60a5563a..b47dec38d6 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -56,6 +56,7 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
+	ThreadFuture<UID> getClusterId() override;
 
 private:
 	friend class ThreadSafeTransaction;

From 5dccca4081b78841d62bfa7c302a8ab8e4e784fe Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 12 Jan 2022 16:49:00 -0500
Subject: [PATCH 035/210] Implement initial concept of shared cache space in
 NativeAPI DatabaseContext

---
 fdbclient/DatabaseContext.h         |  4 ++
 fdbclient/FDBTypes.h                |  6 +++
 fdbclient/MultiVersionTransaction.h |  2 +
 fdbclient/NativeAPI.actor.cpp       | 64 ++++++++++++++++++++++-------
 4 files changed, 62 insertions(+), 14 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index e83b63f260..baabe62f7c 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -436,7 +436,11 @@ public:
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
 	double lastTimedGrv;
 	Version cachedRv;
+	GRVCacheSpace* sharedCachePtr;
 	void updateCachedRV(double t, Version v);
+	Version getCachedRV();
+	double getLastTimedGRV();
+	void setSharedCacheSpace(GRVCacheSpace* p);
 	double lastTimedRkThrottle;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index db52ea26dd..9ade403e52 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1167,6 +1167,12 @@ struct StorageMigrationType {
 	uint32_t type;
 };
 
+struct GRVCacheSpace {
+	Mutex cacheLock;
+	Version cachedRv;
+	double lastTimedGrv;
+};
+
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
 	int pos = locality.find(':');
 	// locality should be either 0 or in the format '<non_empty_string>:<non_empty_string>'
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 10e4809200..048fc74da0 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -532,6 +532,8 @@ public:
 
 	struct LegacyVersionMonitor;
 
+	std::map<UID, GRVCacheSpace> clusterCacheMap;
+
 	// A struct that manages the current connection state of the MultiVersionDatabase. This wraps the underlying
 	// IDatabase object that is currently interacting with the cluster.
 	struct DatabaseState : ThreadSafeReferenceCounted<DatabaseState> {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 37193adcd5..36055a6cb3 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -202,7 +202,23 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
+void updateCachedRVShared(double t, Version v, GRVCacheSpace* s) {
+	MutexHolder mutex(s->cacheLock);
+	TraceEvent("CheckpointCacheUpdateShared")
+	    .detail("Version", v)
+	    .detail("CurTime", t)
+	    .detail("LastVersion", s->cachedRv)
+	    .detail("LastTime", s->lastTimedGrv);
+	s->cachedRv = v;
+	if (t > s->lastTimedGrv) {
+		s->lastTimedGrv = t;
+	}
+}
+
 void DatabaseContext::updateCachedRV(double t, Version v) {
+	if (sharedCachePtr) {
+		return updateCachedRVShared(t, v, sharedCachePtr);
+	}
 	if (v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")
 		    .detail("Version", v)
@@ -220,6 +236,22 @@ void DatabaseContext::updateCachedRV(double t, Version v) {
 	}
 }
 
+Version DatabaseContext::getCachedRV() {
+	if (sharedCachePtr) {
+		MutexHolder mutex(sharedCachePtr->cacheLock);
+		return sharedCachePtr->cachedRv;
+	}
+	return cachedRv;
+}
+
+double DatabaseContext::getLastTimedGRV() {
+	if (sharedCachePtr) {
+		MutexHolder mutex(sharedCachePtr->cacheLock);
+		return sharedCachePtr->lastTimedGrv;
+	}
+	return lastTimedGrv;
+}
+
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
                                                              StorageServerInterface const& ssi,
                                                              LocalityData const& locality) {
@@ -1033,14 +1065,14 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 		loop {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
-			state double lastTime = cx->lastTimedGrv;
+			state double lastTime = cx->getLastTimedGRV();
 			state double lastProxyTime = cx->lastProxyRequest;
 			TraceEvent("BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
 			    .detail("GrvDelay", grvDelay)
-			    .detail("CachedRv", cx->cachedRv)
-			    .detail("CachedTime", cx->lastTimedGrv)
+			    .detail("CachedRv", cx->getCachedRV())
+			    .detail("CachedTime", cx->getLastTimedGRV())
 			    .detail("Gap", curTime - lastTime)
 			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
 			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) ||
@@ -1052,8 +1084,8 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
 					TraceEvent("BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
-					    .detail("CachedRv", cx->cachedRv)
-					    .detail("CachedTime", cx->lastTimedGrv);
+					    .detail("CachedRv", cx->getCachedRV())
+					    .detail("CachedTime", cx->getLastTimedGRV());
 				} catch (Error& e) {
 					TraceEvent("BackgroundGrvUpdaterTxnError").error(e, true);
 					wait(tr.onError(e));
@@ -1296,10 +1328,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0),
-    lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
-    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), sharedCachePtr(nullptr),
+    lastTimedRkThrottle(0.0), lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID),
+    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
+    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -6046,11 +6078,11 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			if (!cx->grvUpdateHandler.isValid()) {
 				cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
 			}
-			if (now() - cx->lastTimedGrv <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && cx->cachedRv != Version(0)) {
-				TraceEvent("DebugGrvUseCache")
-				    .detail("LastRV", cx->cachedRv)
-				    .detail("LastTime", format("%.6f", cx->lastTimedGrv));
-				readVersion = cx->cachedRv;
+			Version rv = cx->getCachedRV();
+			double lastTime = cx->getLastTimedGRV();
+			if (now() - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
+				TraceEvent("DebugGrvUseCache").detail("LastRV", rv).detail("LastTime", format("%.6f", lastTime));
+				readVersion = rv;
 				return readVersion;
 			} // else go through regular GRV path
 		}
@@ -6930,6 +6962,10 @@ Future<UID> DatabaseContext::getClusterId() {
 	return getClusterIdActor(this);
 }
 
+void DatabaseContext::setSharedCacheSpace(GRVCacheSpace* p) {
+	sharedCachePtr = p;
+}
+
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
 	loop {
 		if (self->version.get() < self->desired.get()) {

From bcc69ac1f5ae4723cff2942007c433cc345bbd65 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 18 Jan 2022 13:02:05 -0500
Subject: [PATCH 036/210] Add clusterCacheMap to multiVersionApi and propagate
 shared object through createDatabase

---
 fdbclient/FDBTypes.h                        |  2 ++
 fdbclient/IClientApi.h                      |  3 ++-
 fdbclient/MultiVersionTransaction.actor.cpp | 15 ++++++++++-----
 fdbclient/MultiVersionTransaction.h         |  9 ++++-----
 fdbclient/ThreadSafeTransaction.cpp         |  9 +++++----
 fdbclient/ThreadSafeTransaction.h           |  4 ++--
 6 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 9ade403e52..da11f6a16f 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1171,6 +1171,8 @@ struct GRVCacheSpace {
 	Mutex cacheLock;
 	Version cachedRv;
 	double lastTimedGrv;
+
+	GRVCacheSpace() : cacheLock(Mutex()), cachedRv(Version(0)), lastTimedGrv(0.0) {}
 };
 
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index 86e96c4e3a..2b8827658c 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -153,7 +153,8 @@ public:
 	virtual void runNetwork() = 0;
 	virtual void stopNetwork() = 0;
 
-	virtual Reference<IDatabase> createDatabase(const char* clusterFilePath) = 0;
+	virtual Reference<IDatabase> createDatabase(const char* clusterFilePath,
+	                                            GRVCacheSpace* sharedCachePtr = nullptr) = 0;
 
 	virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) = 0;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index 0bf8a091c6..7b9eb53da8 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -649,10 +649,11 @@ Reference<IDatabase> DLApi::createDatabase609(const char* clusterFilePath) {
 	return makeReference<DLDatabase>(api, dbFuture);
 }
 
-Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath) {
+Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
 	if (headerVersion >= 610) {
 		FdbCApi::FDBDatabase* db;
-		throwIfError(api->createDatabase(clusterFilePath, &db));
+		// can the FdbCApi wrapper signature be changed to add this ptr?
+		throwIfError(api->createDatabase(clusterFilePath, &db, sharedCachePtr));
 		return Reference<IDatabase>(new DLDatabase(api, db));
 	} else {
 		return DLApi::createDatabase609(clusterFilePath);
@@ -1995,13 +1996,17 @@ void MultiVersionApi::addNetworkThreadCompletionHook(void (*hook)(void*), void*
 }
 
 // Creates an IDatabase object that represents a connection to the cluster
-Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath) {
+Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
 	lock.enter();
 	if (!networkSetup) {
 		lock.leave();
 		throw network_not_setup();
 	}
 	std::string clusterFile(clusterFilePath);
+	if (clusterCacheMap.find(clusterFile) == clusterCacheMap.end()) {
+		clusterCacheMap[clusterFile] = GRVCacheSpace();
+	}
+	sharedCachePtr = &clusterCacheMap[clusterFile];
 
 	if (localClientDisabled) {
 		ASSERT(!bypassMultiClientApi);
@@ -2010,7 +2015,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 		nextThread = (nextThread + 1) % threadCount;
 		lock.leave();
 
-		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
+		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath, sharedCachePtr);
 		return Reference<IDatabase>(
 		    new MultiVersionDatabase(this, threadIdx, clusterFile, Reference<IDatabase>(), localDb));
 	}
@@ -2019,7 +2024,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 
 	ASSERT_LE(threadCount, 1);
 
-	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
+	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath, sharedCachePtr);
 	if (bypassMultiClientApi) {
 		return localDb;
 	} else {
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 048fc74da0..5ed0ac04e8 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -62,7 +62,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	fdb_error_t (*setupNetwork)();
 	fdb_error_t (*runNetwork)();
 	fdb_error_t (*stopNetwork)();
-	fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db);
+	fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db, GRVCacheSpace* sharedCachePtr);
 
 	// Database
 	fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr);
@@ -331,7 +331,7 @@ public:
 	void runNetwork() override;
 	void stopNetwork() override;
 
-	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
 	Reference<IDatabase> createDatabase609(const char* clusterFilePath); // legacy database creation
 
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
@@ -532,8 +532,6 @@ public:
 
 	struct LegacyVersionMonitor;
 
-	std::map<UID, GRVCacheSpace> clusterCacheMap;
-
 	// A struct that manages the current connection state of the MultiVersionDatabase. This wraps the underlying
 	// IDatabase object that is currently interacting with the cluster.
 	struct DatabaseState : ThreadSafeReferenceCounted<DatabaseState> {
@@ -639,7 +637,7 @@ public:
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
 
 	// Creates an IDatabase object that represents a connection to the cluster
-	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
 	static MultiVersionApi* api;
 
 	Reference<ClientInfo> getLocalClient();
@@ -675,6 +673,7 @@ private:
 	Reference<ClientInfo> localClient;
 	std::map<std::string, ClientDesc> externalClientDescriptions;
 	std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
+	std::map<std::string, GRVCacheSpace> clusterCacheMap;
 
 	bool networkStartSetup;
 	volatile bool networkSetup;
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index c14f768be4..9f2942499d 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -115,7 +115,7 @@ ThreadFuture<ProtocolVersion> ThreadSafeDatabase::getServerProtocol(Optional<Pro
 	    [db, expectedVersion]() -> Future<ProtocolVersion> { return db->getClusterProtocol(expectedVersion); });
 }
 
-ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) {
+ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion, GRVCacheSpace* sharedCachePtr) {
 	ClusterConnectionFile* connFile =
 	    new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first);
 
@@ -124,7 +124,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion)
 	DatabaseContext* db = this->db = DatabaseContext::allocateOnForeignThread();
 
 	onMainThreadVoid(
-	    [db, connFile, apiVersion]() {
+	    [db, connFile, apiVersion, sharedCachePtr]() {
 		    try {
 			    Database::createDatabase(
 			        Reference<ClusterConnectionFile>(connFile), apiVersion, IsInternal::False, LocalityData(), db)
@@ -134,6 +134,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion)
 		    } catch (...) {
 			    new (db) DatabaseContext(unknown_error());
 		    }
+		    db->setSharedCacheSpace(sharedCachePtr);
 	    },
 	    nullptr);
 }
@@ -499,8 +500,8 @@ void ThreadSafeApi::stopNetwork() {
 	::stopNetwork();
 }
 
-Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath) {
-	return Reference<IDatabase>(new ThreadSafeDatabase(clusterFilePath, apiVersion));
+Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
+	return Reference<IDatabase>(new ThreadSafeDatabase(clusterFilePath, apiVersion, sharedCachePtr));
 }
 
 void ThreadSafeApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) {
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index b47dec38d6..81b73ff8e1 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -64,7 +64,7 @@ private:
 	DatabaseContext* db;
 
 public: // Internal use only
-	ThreadSafeDatabase(std::string connFilename, int apiVersion);
+	ThreadSafeDatabase(std::string connFilename, int apiVersion, GRVCacheSpace* sharedCachePtr = nullptr);
 	ThreadSafeDatabase(DatabaseContext* db) : db(db) {}
 	DatabaseContext* unsafeGetPtr() const { return db; }
 };
@@ -169,7 +169,7 @@ public:
 	void runNetwork() override;
 	void stopNetwork() override;
 
-	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
 
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
 

From ca1839c405a7b877dfb0a908665402e0765ca26e Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 20 Jan 2022 15:45:50 -0500
Subject: [PATCH 037/210] add sim_validation back to check staleness

---
 fdbclient/NativeAPI.actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 36055a6cb3..43e101ba6a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6080,8 +6080,10 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			}
 			Version rv = cx->getCachedRV();
 			double lastTime = cx->getLastTimedGRV();
-			if (now() - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
+			double requestTime = now();
+			if (requestTime - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
 				TraceEvent("DebugGrvUseCache").detail("LastRV", rv).detail("LastTime", format("%.6f", lastTime));
+				ASSERT(!debug_checkVersionTime(rv, requestTime, "CheckStaleness"));
 				readVersion = rv;
 				return readVersion;
 			} // else go through regular GRV path

From 3a21438bb82c3908a521b331e730a237df5c45f9 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 21 Jan 2022 13:12:30 -0500
Subject: [PATCH 038/210] remove getclusterid api

---
 bindings/c/fdb_c.cpp                        |  4 ----
 bindings/c/foundationdb/fdb_c.h             |  2 --
 fdbclient/DatabaseContext.h                 |  2 --
 fdbclient/IClientApi.h                      |  2 --
 fdbclient/MultiVersionTransaction.actor.cpp | 26 ---------------------
 fdbclient/MultiVersionTransaction.h         |  3 ---
 fdbclient/NativeAPI.actor.cpp               | 20 ----------------
 fdbclient/ThreadSafeTransaction.cpp         |  5 ----
 fdbclient/ThreadSafeTransaction.h           |  1 -
 9 files changed, 65 deletions(-)

diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index 1a25c86b02..612f454125 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -405,10 +405,6 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
 	                        .extractPtr());
 }
 
-extern "C" DLLEXPORT FDBFuture* fdb_database_get_cluster_id(FDBDatabase* db) {
-	return (FDBFuture*)(DB(db)->getClusterId().extractPtr());
-}
-
 // Get network thread busyness (updated every 1s)
 // A value of 0 indicates that the client is more or less idle
 // A value of 1 (or more) indicates that the client is saturated
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index 43b875f889..9779582b75 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -226,8 +226,6 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_create_snapshot(FDBDatabase
                                                                      uint8_t const* snap_command,
                                                                      int snap_command_length);
 
-DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_cluster_id(FDBDatabase* db);
-
 DLLEXPORT WARN_UNUSED_RESULT double fdb_database_get_main_thread_busyness(FDBDatabase* db);
 
 DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db, uint64_t expected_version);
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 2c378550b3..55e0c91d41 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -319,8 +319,6 @@ public:
 	// Management API, create snapshot
 	Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
 
-	Future<UID> getClusterId();
-
 	Future<Void> getChangeFeedStream(Reference<ChangeFeedData> results,
 	                                 Key rangeID,
 	                                 Version begin = 0,
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index cd1919c412..def5ba2e2e 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -137,8 +137,6 @@ public:
 	// Management API, create snapshot
 	virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
 
-	virtual ThreadFuture<UID> getClusterId() = 0;
-
 	// used in template functions as the Transaction type that can be created through createTransaction()
 	using TransactionT = ITransaction;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index d4e10f89ac..0934c48745 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -455,25 +455,6 @@ double DLDatabase::getMainThreadBusyness() {
 	return 0;
 }
 
-ThreadFuture<UID> DLDatabase::getClusterId() {
-	if (api->getClusterId != nullptr) {
-		FdbCApi::FDBFuture* f = api->getClusterId();
-		return toThreadFuture<UID>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
-			const uint8_t* res;
-			int resLen;
-			FdbCApi::fdb_error_t error = api->futureGetKey(f, &res, &resLen);
-			ASSERT(!error);
-			// UID loading code taken from
-			// IRandom.h:load(const uint8_t* i, UID& out, Context& context)
-			const uint64_t* in = reinterpret_cast<const uint64_t*>(res);
-			UID out = UID(in[0], in[1]);
-			return out;
-		});
-	}
-
-	return UID();
-}
-
 // Returns the protocol version reported by the coordinator this client is connected to
 // If an expected version is given, the future won't return until the protocol version is different than expected
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
@@ -1321,13 +1302,6 @@ double MultiVersionDatabase::getMainThreadBusyness() {
 	return localClientBusyness;
 }
 
-ThreadFuture<UID> MultiVersionDatabase::getClusterId() {
-	if (dbState->db) {
-		return dbState->db->getClusterId();
-	}
-	return UID();
-}
-
 // Returns the protocol version reported by the coordinator this client is connected to
 // If an expected version is given, the future won't return until the protocol version is different than expected
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 77f57b9a53..a1ee26b40b 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -110,7 +110,6 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                     int snapshotCommandLength);
 	double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
 	FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
-	FDBFuture* (*getClusterId)();
 
 	// Transaction
 	fdb_error_t (*transactionSetOption)(FDBTransaction* tr,
@@ -359,7 +358,6 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
-	ThreadFuture<UID> getClusterId() override;
 
 private:
 	const Reference<FdbCApi> api;
@@ -588,7 +586,6 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
-	ThreadFuture<UID> getClusterId() override;
 
 	// private:
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a68eb3fe5c..b66eca0e4f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7162,26 +7162,6 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
 	return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
 }
 
-ACTOR static Future<UID> getClusterIdActor(DatabaseContext* cx) {
-	Database db(cx);
-	state Transaction tr(db);
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> clusterId = wait(tr.get(clusterIdKey));
-			ASSERT(clusterId.present());
-			return BinaryReader::fromStringRef<UID>(clusterId.get(), Unversioned());
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-}
-
-Future<UID> DatabaseContext::getClusterId() {
-	return getClusterIdActor(this);
-}
-
 void DatabaseContext::setSharedCacheSpace(GRVCacheSpace* p) {
 	sharedCachePtr = p;
 }
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index 0107553cdd..36a38cb6ad 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -96,11 +96,6 @@ ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons
 	return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
 }
 
-ThreadFuture<UID> ThreadSafeDatabase::getClusterId() {
-	DatabaseContext* db = this->db;
-	return onMainThread([db]() -> Future<UID> { return db->getClusterId(); });
-}
-
 // Return the main network thread busyness
 double ThreadSafeDatabase::getMainThreadBusyness() {
 	ASSERT(g_network);
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index 747e66ebac..523eb1c85a 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -56,7 +56,6 @@ public:
 	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
-	ThreadFuture<UID> getClusterId() override;
 
 private:
 	friend class ThreadSafeTransaction;

From dd1d48754d8ca5f0aad5282b97aeac5e55b1b25b Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 21 Jan 2022 14:32:04 -0500
Subject: [PATCH 039/210] revert create_database signatures and added new
 shared state struct

---
 fdbclient/DatabaseContext.h                 |  7 ++-
 fdbclient/FDBTypes.h                        | 11 ++++-
 fdbclient/IClientApi.h                      |  3 +-
 fdbclient/MultiVersionTransaction.actor.cpp | 14 +++---
 fdbclient/MultiVersionTransaction.h         |  6 +--
 fdbclient/NativeAPI.actor.cpp               | 47 ++++++++++++---------
 fdbclient/ThreadSafeTransaction.cpp         |  9 ++--
 fdbclient/ThreadSafeTransaction.h           |  4 +-
 8 files changed, 56 insertions(+), 45 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 55e0c91d41..d4e1830219 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -469,15 +469,18 @@ public:
 	int outstandingWatches;
 	int maxOutstandingWatches;
 
+	// Manage any shared state that may be used by MVC
+	std::shared_ptr<DatabaseSharedState> sharedStatePtr;
+	std::shared_ptr<DatabaseSharedState> initSharedState();
+	void setSharedState(std::shared_ptr<DatabaseSharedState> p);
+
 	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
 	double lastTimedGrv;
 	Version cachedRv;
-	GRVCacheSpace* sharedCachePtr;
 	void updateCachedRV(double t, Version v);
 	Version getCachedRV();
 	double getLastTimedGRV();
-	void setSharedCacheSpace(GRVCacheSpace* p);
 	double lastTimedRkThrottle;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index b5958045d6..995f17a660 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1168,11 +1168,18 @@ struct StorageMigrationType {
 };
 
 struct GRVCacheSpace {
-	Mutex cacheLock;
 	Version cachedRv;
 	double lastTimedGrv;
 
-	GRVCacheSpace() : cacheLock(Mutex()), cachedRv(Version(0)), lastTimedGrv(0.0) {}
+	GRVCacheSpace() : cachedRv(Version(0)), lastTimedGrv(0.0) {}
+};
+
+// This structure can be extended in the future to include additional features that required a shared state
+struct DatabaseSharedState {
+	Mutex mutexLock;
+	GRVCacheSpace grvCacheSpace;
+
+	DatabaseSharedState() : mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()) {}
 };
 
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index def5ba2e2e..f62d588591 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -158,8 +158,7 @@ public:
 	virtual void runNetwork() = 0;
 	virtual void stopNetwork() = 0;
 
-	virtual Reference<IDatabase> createDatabase(const char* clusterFilePath,
-	                                            GRVCacheSpace* sharedCachePtr = nullptr) = 0;
+	virtual Reference<IDatabase> createDatabase(const char* clusterFilePath) = 0;
 
 	virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) = 0;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index 0934c48745..55d92ba17c 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -720,11 +720,11 @@ Reference<IDatabase> DLApi::createDatabase609(const char* clusterFilePath) {
 	return makeReference<DLDatabase>(api, dbFuture);
 }
 
-Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
+Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath) {
 	if (headerVersion >= 610) {
 		FdbCApi::FDBDatabase* db;
 		// can the FdbCApi wrapper signature be changed to add this ptr?
-		throwIfError(api->createDatabase(clusterFilePath, &db, sharedCachePtr));
+		throwIfError(api->createDatabase(clusterFilePath, &db));
 		return Reference<IDatabase>(new DLDatabase(api, db));
 	} else {
 		return DLApi::createDatabase609(clusterFilePath);
@@ -2105,17 +2105,13 @@ void MultiVersionApi::addNetworkThreadCompletionHook(void (*hook)(void*), void*
 }
 
 // Creates an IDatabase object that represents a connection to the cluster
-Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
+Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath) {
 	lock.enter();
 	if (!networkSetup) {
 		lock.leave();
 		throw network_not_setup();
 	}
 	std::string clusterFile(clusterFilePath);
-	if (clusterCacheMap.find(clusterFile) == clusterCacheMap.end()) {
-		clusterCacheMap[clusterFile] = GRVCacheSpace();
-	}
-	sharedCachePtr = &clusterCacheMap[clusterFile];
 
 	if (localClientDisabled) {
 		ASSERT(!bypassMultiClientApi);
@@ -2124,7 +2120,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 		nextThread = (nextThread + 1) % threadCount;
 		lock.leave();
 
-		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath, sharedCachePtr);
+		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
 		return Reference<IDatabase>(
 		    new MultiVersionDatabase(this, threadIdx, clusterFile, Reference<IDatabase>(), localDb));
 	}
@@ -2133,7 +2129,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 
 	ASSERT_LE(threadCount, 1);
 
-	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath, sharedCachePtr);
+	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
 	if (bypassMultiClientApi) {
 		return localDb;
 	} else {
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index a1ee26b40b..ccb65d24b3 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -88,7 +88,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	fdb_error_t (*setupNetwork)();
 	fdb_error_t (*runNetwork)();
 	fdb_error_t (*stopNetwork)();
-	fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db, GRVCacheSpace* sharedCachePtr);
+	fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db);
 
 	// Database
 	fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr);
@@ -380,7 +380,7 @@ public:
 	void runNetwork() override;
 	void stopNetwork() override;
 
-	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
 	Reference<IDatabase> createDatabase609(const char* clusterFilePath); // legacy database creation
 
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
@@ -696,7 +696,7 @@ public:
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
 
 	// Creates an IDatabase object that represents a connection to the cluster
-	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
 	static MultiVersionApi* api;
 
 	Reference<ClientInfo> getLocalClient();
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b66eca0e4f..b93f2c3c5b 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -209,22 +209,22 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-void updateCachedRVShared(double t, Version v, GRVCacheSpace* s) {
-	MutexHolder mutex(s->cacheLock);
+void updateCachedRVShared(double t, Version v, std::shared_ptr<DatabaseSharedState> p) {
+	MutexHolder mutex(p->mutexLock);
 	TraceEvent("CheckpointCacheUpdateShared")
 	    .detail("Version", v)
 	    .detail("CurTime", t)
-	    .detail("LastVersion", s->cachedRv)
-	    .detail("LastTime", s->lastTimedGrv);
-	s->cachedRv = v;
-	if (t > s->lastTimedGrv) {
-		s->lastTimedGrv = t;
+	    .detail("LastVersion", p->grvCacheSpace.cachedRv)
+	    .detail("LastTime", p->grvCacheSpace.lastTimedGrv);
+	p->grvCacheSpace.cachedRv = v;
+	if (t > p->grvCacheSpace.lastTimedGrv) {
+		p->grvCacheSpace.lastTimedGrv = t;
 	}
 }
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (sharedCachePtr) {
-		return updateCachedRVShared(t, v, sharedCachePtr);
+	if (sharedStatePtr) {
+		return updateCachedRVShared(t, v, sharedStatePtr);
 	}
 	if (v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")
@@ -244,17 +244,17 @@ void DatabaseContext::updateCachedRV(double t, Version v) {
 }
 
 Version DatabaseContext::getCachedRV() {
-	if (sharedCachePtr) {
-		MutexHolder mutex(sharedCachePtr->cacheLock);
-		return sharedCachePtr->cachedRv;
+	if (sharedStatePtr) {
+		MutexHolder mutex(sharedStatePtr->mutexLock);
+		return sharedStatePtr->grvCacheSpace.cachedRv;
 	}
 	return cachedRv;
 }
 
 double DatabaseContext::getLastTimedGRV() {
-	if (sharedCachePtr) {
-		MutexHolder mutex(sharedCachePtr->cacheLock);
-		return sharedCachePtr->lastTimedGrv;
+	if (sharedStatePtr) {
+		MutexHolder mutex(sharedStatePtr->mutexLock);
+		return sharedStatePtr->grvCacheSpace.lastTimedGrv;
 	}
 	return lastTimedGrv;
 }
@@ -1335,7 +1335,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), sharedCachePtr(nullptr),
+    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), sharedStatePtr(nullptr),
     lastTimedRkThrottle(0.0), lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID),
     clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
     mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
@@ -6052,7 +6052,7 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 		return true;
 	}
 	TraceEvent("DebugGrvRkCd")
-		.detail("LastTimedRkThrottle", cx->lastTimedRkThrottle)
+	    .detail("LastTimedRkThrottle", cx->lastTimedRkThrottle)
 	    .detail("TimeElapsed", now() - cx->lastTimedRkThrottle)
 	    .detail("CooldownTime", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
@@ -6061,7 +6061,8 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	if (!readVersion.isValid()) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !trState->options.skipGrvCache &&
-		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || trState->options.useGrvCache) &&
+		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE ||
+		     trState->options.useGrvCache) &&
 		    rkThrottlingCooledDown(getDatabase().getPtr())) {
 			TraceEvent("DebugGrvEnterCachePath");
 			// Upon our first request to use cached RVs, start the background updater
@@ -7162,8 +7163,14 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
 	return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
 }
 
-void DatabaseContext::setSharedCacheSpace(GRVCacheSpace* p) {
-	sharedCachePtr = p;
+std::shared_ptr<DatabaseSharedState> DatabaseContext::initSharedState() {
+	std::shared_ptr<DatabaseSharedState> newState = std::make_shared<DatabaseSharedState>();
+	setSharedState(newState);
+	return newState;
+}
+
+void DatabaseContext::setSharedState(std::shared_ptr<DatabaseSharedState> p) {
+	sharedStatePtr = p;
 }
 
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index 36a38cb6ad..eb50b0f16f 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -111,7 +111,7 @@ ThreadFuture<ProtocolVersion> ThreadSafeDatabase::getServerProtocol(Optional<Pro
 	    [db, expectedVersion]() -> Future<ProtocolVersion> { return db->getClusterProtocol(expectedVersion); });
 }
 
-ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion, GRVCacheSpace* sharedCachePtr) {
+ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) {
 	ClusterConnectionFile* connFile =
 	    new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first);
 
@@ -120,7 +120,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion,
 	DatabaseContext* db = this->db = DatabaseContext::allocateOnForeignThread();
 
 	onMainThreadVoid(
-	    [db, connFile, apiVersion, sharedCachePtr]() {
+	    [db, connFile, apiVersion]() {
 		    try {
 			    Database::createDatabase(
 			        Reference<ClusterConnectionFile>(connFile), apiVersion, IsInternal::False, LocalityData(), db)
@@ -130,7 +130,6 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion,
 		    } catch (...) {
 			    new (db) DatabaseContext(unknown_error());
 		    }
-		    db->setSharedCacheSpace(sharedCachePtr);
 	    },
 	    nullptr);
 }
@@ -544,8 +543,8 @@ void ThreadSafeApi::stopNetwork() {
 	::stopNetwork();
 }
 
-Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr) {
-	return Reference<IDatabase>(new ThreadSafeDatabase(clusterFilePath, apiVersion, sharedCachePtr));
+Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath) {
+	return Reference<IDatabase>(new ThreadSafeDatabase(clusterFilePath, apiVersion));
 }
 
 void ThreadSafeApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) {
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index 523eb1c85a..f61e0de2e8 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -63,7 +63,7 @@ private:
 	DatabaseContext* db;
 
 public: // Internal use only
-	ThreadSafeDatabase(std::string connFilename, int apiVersion, GRVCacheSpace* sharedCachePtr = nullptr);
+	ThreadSafeDatabase(std::string connFilename, int apiVersion);
 	ThreadSafeDatabase(DatabaseContext* db) : db(db) {}
 	DatabaseContext* unsafeGetPtr() const { return db; }
 };
@@ -175,7 +175,7 @@ public:
 	void runNetwork() override;
 	void stopNetwork() override;
 
-	Reference<IDatabase> createDatabase(const char* clusterFilePath, GRVCacheSpace* sharedCachePtr = nullptr) override;
+	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
 
 	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
 

From 133ee2c11309268f3100202bb99f6aef66f388bc Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 21 Jan 2022 16:52:11 -0500
Subject: [PATCH 040/210] added fdb_c_internal.h and set up shared space
 initialization in multiversion api db creation

---
 bindings/c/CMakeLists.txt                   |  1 +
 bindings/c/fdb_c.cpp                        |  9 ++++
 bindings/c/foundationdb/fdb_c.h             |  3 ++
 bindings/c/foundationdb/fdb_c_internal.h    | 55 +++++++++++++++++++++
 fdbclient/DatabaseContext.h                 |  6 +--
 fdbclient/IClientApi.h                      |  4 ++
 fdbclient/MultiVersionTransaction.actor.cpp | 40 +++++++++++++++
 fdbclient/MultiVersionTransaction.h         | 11 ++++-
 fdbclient/NativeAPI.actor.cpp               |  8 +--
 fdbclient/ThreadSafeTransaction.cpp         |  8 +++
 fdbclient/ThreadSafeTransaction.h           |  3 ++
 11 files changed, 140 insertions(+), 8 deletions(-)
 create mode 100644 bindings/c/foundationdb/fdb_c_internal.h

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index d08bb48344..09fa449579 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(FDB_C_SRCS
   fdb_c.cpp
   foundationdb/fdb_c.h
+  foundationdb/fdb_c_internal.h
   ThreadCleanup.cpp)
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index 612f454125..9ca6277dd2 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -26,6 +26,7 @@
 #include "fdbclient/MultiVersionTransaction.h"
 #include "fdbclient/MultiVersionAssignmentVars.h"
 #include "foundationdb/fdb_c.h"
+#include "foundationdb/fdb_c_internal.h"
 
 int g_api_version = 0;
 
@@ -405,6 +406,14 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
 	                        .extractPtr());
 }
 
+extern "C" DLLEXPORT DatabaseSharedState* fdb_database_create_shared_state(FDBDatabase* db) {
+	return (DatabaseSharedState*)(DB(db)->createSharedState());
+}
+
+extern "C" DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p) {
+	(DB(db)->setSharedState(p));
+}
+
 // Get network thread busyness (updated every 1s)
 // A value of 0 indicates that the client is more or less idle
 // A value of 1 (or more) indicates that the client is saturated
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index 9779582b75..1246fd71c1 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -63,6 +63,8 @@
 extern "C" {
 #endif
 
+#ifndef FDB_API_OBJECTS
+#define FDB_API_OBJECTS
 /* Pointers to these opaque types represent objects in the FDB API */
 typedef struct FDB_future FDBFuture;
 typedef struct FDB_result FDBResult;
@@ -71,6 +73,7 @@ typedef struct FDB_transaction FDBTransaction;
 
 typedef int fdb_error_t;
 typedef int fdb_bool_t;
+#endif
 
 DLLEXPORT const char* fdb_get_error(fdb_error_t code);
 
diff --git a/bindings/c/foundationdb/fdb_c_internal.h b/bindings/c/foundationdb/fdb_c_internal.h
new file mode 100644
index 0000000000..019560ecad
--- /dev/null
+++ b/bindings/c/foundationdb/fdb_c_internal.h
@@ -0,0 +1,55 @@
+/*
+ * fdb_c_internal.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDB_C_INTERNAL_H
+#define FDB_C_INTERNAL_H
+#pragma once
+
+#ifndef DLLEXPORT
+#define DLLEXPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef FDB_API_OBJECTS
+#define FDB_API_OBJECTS
+/* Pointers to these opaque types represent objects in the FDB API */
+typedef struct FDB_future FDBFuture;
+typedef struct FDB_result FDBResult;
+typedef struct FDB_database FDBDatabase;
+typedef struct FDB_transaction FDBTransaction;
+
+typedef int fdb_error_t;
+typedef int fdb_bool_t;
+#endif
+
+// forward declaration and typedef
+typedef struct DatabaseSharedState DatabaseSharedState;
+
+DLLEXPORT DatabaseSharedState* fdb_database_create_shared_state(FDBDatabase* db);
+
+DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index d4e1830219..88d4f6ccf1 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -470,9 +470,9 @@ public:
 	int maxOutstandingWatches;
 
 	// Manage any shared state that may be used by MVC
-	std::shared_ptr<DatabaseSharedState> sharedStatePtr;
-	std::shared_ptr<DatabaseSharedState> initSharedState();
-	void setSharedState(std::shared_ptr<DatabaseSharedState> p);
+	DatabaseSharedState* sharedStatePtr;
+	DatabaseSharedState* initSharedState();
+	void setSharedState(DatabaseSharedState* p);
 
 	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index f62d588591..aa9b24576c 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -137,6 +137,10 @@ public:
 	// Management API, create snapshot
 	virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
 
+	// Interface to manage shared state across multiple connections to the same Database
+	virtual DatabaseSharedState* createSharedState() = 0;
+	virtual void setSharedState(DatabaseSharedState* p) = 0;
+
 	// used in template functions as the Transaction type that can be created through createTransaction()
 	using TransactionT = ITransaction;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index 55d92ba17c..53b732c2bf 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -446,6 +446,19 @@ ThreadFuture<Void> DLDatabase::createSnapshot(const StringRef& uid, const String
 	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
 }
 
+DatabaseSharedState* DLDatabase::createSharedState() {
+	if (!api->databaseCreateSharedState) {
+		return nullptr;
+	}
+	return api->databaseCreateSharedState(db);
+}
+
+void DLDatabase::setSharedState(DatabaseSharedState* p) {
+	if (api->databaseSetSharedState) {
+		api->databaseSetSharedState(db, p);
+	}
+}
+
 // Get network thread busyness
 double DLDatabase::getMainThreadBusyness() {
 	if (api->databaseGetMainThreadBusyness != nullptr) {
@@ -1288,6 +1301,19 @@ ThreadFuture<Void> MultiVersionDatabase::createSnapshot(const StringRef& uid, co
 	return abortableFuture(f, dbState->dbVar->get().onChange);
 }
 
+DatabaseSharedState* MultiVersionDatabase::createSharedState() {
+	if (dbState->db) {
+		return dbState->db->createSharedState();
+	}
+	return nullptr;
+}
+
+void MultiVersionDatabase::setSharedState(DatabaseSharedState* p) {
+	if (dbState->db) {
+		dbState->db->setSharedState(p);
+	}
+}
+
 // Get network thread busyness
 // Return the busyness for the main thread. When using external clients, take the larger of the local client
 // and the external client's busyness.
@@ -2121,6 +2147,13 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 		lock.leave();
 
 		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
+		if (clusterSharedStateMap.find(clusterFile) == clusterSharedStateMap.end() ||
+		    clusterSharedStateMap[clusterFile] == nullptr) {
+			DatabaseSharedState* p = localDb->createSharedState();
+			clusterSharedStateMap[clusterFile] = p;
+		} else {
+			localDb->setSharedState(clusterSharedStateMap[clusterFile]);
+		}
 		return Reference<IDatabase>(
 		    new MultiVersionDatabase(this, threadIdx, clusterFile, Reference<IDatabase>(), localDb));
 	}
@@ -2130,6 +2163,13 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 	ASSERT_LE(threadCount, 1);
 
 	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
+	if (clusterSharedStateMap.find(clusterFile) == clusterSharedStateMap.end() ||
+	    clusterSharedStateMap[clusterFile] == nullptr) {
+		DatabaseSharedState* p = localDb->createSharedState();
+		clusterSharedStateMap[clusterFile] = p;
+	} else {
+		localDb->setSharedState(clusterSharedStateMap[clusterFile]);
+	}
 	if (bypassMultiClientApi) {
 		return localDb;
 	} else {
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index ccb65d24b3..9896bff0f1 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -108,6 +108,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                     int uidLength,
 	                                     uint8_t const* snapshotCommmand,
 	                                     int snapshotCommandLength);
+	DatabaseSharedState* (*databaseCreateSharedState)(FDBDatabase* database);
+	void (*databaseSetSharedState)(FDBDatabase* database, DatabaseSharedState* p);
+
 	double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
 	FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
 
@@ -359,6 +362,9 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
+	DatabaseSharedState* createSharedState() override;
+	void setSharedState(DatabaseSharedState* p) override;
+
 private:
 	const Reference<FdbCApi> api;
 	FdbCApi::FDBDatabase*
@@ -587,6 +593,9 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
+	DatabaseSharedState* createSharedState() override;
+	void setSharedState(DatabaseSharedState* p) override;
+
 	// private:
 
 	struct LegacyVersionMonitor;
@@ -732,7 +741,7 @@ private:
 	Reference<ClientInfo> localClient;
 	std::map<std::string, ClientDesc> externalClientDescriptions;
 	std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
-	std::map<std::string, GRVCacheSpace> clusterCacheMap;
+	std::map<std::string, DatabaseSharedState*> clusterSharedStateMap;
 
 	bool networkStartSetup;
 	volatile bool networkSetup;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b93f2c3c5b..c13636b6f1 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -209,7 +209,7 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-void updateCachedRVShared(double t, Version v, std::shared_ptr<DatabaseSharedState> p) {
+void updateCachedRVShared(double t, Version v, DatabaseSharedState* p) {
 	MutexHolder mutex(p->mutexLock);
 	TraceEvent("CheckpointCacheUpdateShared")
 	    .detail("Version", v)
@@ -7163,13 +7163,13 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
 	return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
 }
 
-std::shared_ptr<DatabaseSharedState> DatabaseContext::initSharedState() {
-	std::shared_ptr<DatabaseSharedState> newState = std::make_shared<DatabaseSharedState>();
+DatabaseSharedState* DatabaseContext::initSharedState() {
+	DatabaseSharedState* newState = new DatabaseSharedState();
 	setSharedState(newState);
 	return newState;
 }
 
-void DatabaseContext::setSharedState(std::shared_ptr<DatabaseSharedState> p) {
+void DatabaseContext::setSharedState(DatabaseSharedState* p) {
 	sharedStatePtr = p;
 }
 
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index eb50b0f16f..ad6032e454 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -96,6 +96,14 @@ ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons
 	return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
 }
 
+DatabaseSharedState* ThreadSafeDatabase::createSharedState() {
+	return db->initSharedState();
+}
+
+void ThreadSafeDatabase::setSharedState(DatabaseSharedState* p) {
+	db->setSharedState(p);
+}
+
 // Return the main network thread busyness
 double ThreadSafeDatabase::getMainThreadBusyness() {
 	ASSERT(g_network);
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index f61e0de2e8..b93e9483a5 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -57,6 +57,9 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
+	DatabaseSharedState* createSharedState() override;
+	void setSharedState(DatabaseSharedState* p) override;
+
 private:
 	friend class ThreadSafeTransaction;
 	bool isConfigDB{ false };

From 9db49484d4d4477b93c06b7a375683c0236403e3 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 25 Jan 2022 15:56:58 -0500
Subject: [PATCH 041/210] add reference counting to shared state and create new
 fdb_c_types header

---
 bindings/c/CMakeLists.txt                |  1 +
 bindings/c/foundationdb/fdb_c.h          | 13 +------
 bindings/c/foundationdb/fdb_c_internal.h | 14 ++------
 bindings/c/foundationdb/fdb_c_types.h    | 45 ++++++++++++++++++++++++
 fdbclient/FDBTypes.h                     |  1 +
 fdbclient/NativeAPI.actor.cpp            |  8 +++++
 6 files changed, 58 insertions(+), 24 deletions(-)
 create mode 100644 bindings/c/foundationdb/fdb_c_types.h

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 09fa449579..c84f46b33c 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -2,6 +2,7 @@ set(FDB_C_SRCS
   fdb_c.cpp
   foundationdb/fdb_c.h
   foundationdb/fdb_c_internal.h
+  foundationdb/fdb_c_types.h
   ThreadCleanup.cpp)
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index 1246fd71c1..e529bfcfe8 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -58,23 +58,12 @@
 #include <stdint.h>
 
 #include "fdb_c_options.g.h"
+#include "fdb_c_types.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef FDB_API_OBJECTS
-#define FDB_API_OBJECTS
-/* Pointers to these opaque types represent objects in the FDB API */
-typedef struct FDB_future FDBFuture;
-typedef struct FDB_result FDBResult;
-typedef struct FDB_database FDBDatabase;
-typedef struct FDB_transaction FDBTransaction;
-
-typedef int fdb_error_t;
-typedef int fdb_bool_t;
-#endif
-
 DLLEXPORT const char* fdb_get_error(fdb_error_t code);
 
 DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);
diff --git a/bindings/c/foundationdb/fdb_c_internal.h b/bindings/c/foundationdb/fdb_c_internal.h
index 019560ecad..15312222ce 100644
--- a/bindings/c/foundationdb/fdb_c_internal.h
+++ b/bindings/c/foundationdb/fdb_c_internal.h
@@ -26,22 +26,12 @@
 #define DLLEXPORT
 #endif
 
+#include "fdb_c_types.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef FDB_API_OBJECTS
-#define FDB_API_OBJECTS
-/* Pointers to these opaque types represent objects in the FDB API */
-typedef struct FDB_future FDBFuture;
-typedef struct FDB_result FDBResult;
-typedef struct FDB_database FDBDatabase;
-typedef struct FDB_transaction FDBTransaction;
-
-typedef int fdb_error_t;
-typedef int fdb_bool_t;
-#endif
-
 // forward declaration and typedef
 typedef struct DatabaseSharedState DatabaseSharedState;
 
diff --git a/bindings/c/foundationdb/fdb_c_types.h b/bindings/c/foundationdb/fdb_c_types.h
new file mode 100644
index 0000000000..64a7771d41
--- /dev/null
+++ b/bindings/c/foundationdb/fdb_c_types.h
@@ -0,0 +1,45 @@
+/*
+ * fdb_c_types.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDB_C_TYPES_H
+#define FDB_C_TYPES_H
+#pragma once
+
+#ifndef DLLEXPORT
+#define DLLEXPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Pointers to these opaque types represent objects in the FDB API */
+typedef struct FDB_future FDBFuture;
+typedef struct FDB_result FDBResult;
+typedef struct FDB_database FDBDatabase;
+typedef struct FDB_transaction FDBTransaction;
+
+typedef int fdb_error_t;
+typedef int fdb_bool_t;
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index b021f90916..59ecc3f072 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1182,6 +1182,7 @@ struct GRVCacheSpace {
 struct DatabaseSharedState {
 	Mutex mutexLock;
 	GRVCacheSpace grvCacheSpace;
+	int refCount;
 
 	DatabaseSharedState() : mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()) {}
 };
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b785e2867f..0d503f82a8 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1627,6 +1627,13 @@ DatabaseContext::~DatabaseContext() {
 	if (grvUpdateHandler.isValid()) {
 		grvUpdateHandler.cancel();
 	}
+	if (sharedStatePtr) {
+		sharedStatePtr->refCount--;
+		if (sharedStatePtr->refCount <= 0) {
+			delete sharedStatePtr;
+			sharedStatePtr = nullptr;
+		}
+	}
 	for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it))
 		it->second->notifyContextDestroyed();
 	ASSERT_ABORT(server_interf.empty());
@@ -7171,6 +7178,7 @@ DatabaseSharedState* DatabaseContext::initSharedState() {
 
 void DatabaseContext::setSharedState(DatabaseSharedState* p) {
 	sharedStatePtr = p;
+	sharedStatePtr->refCount++;
 }
 
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {

From 91ecc4104d8fa656acf26ef6b2b930e0596f11e0 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 28 Jan 2022 16:56:14 -0500
Subject: [PATCH 042/210] change shared state map and update locations

---
 fdbclient/MultiVersionTransaction.actor.cpp | 32 +++++++++++----------
 fdbclient/MultiVersionTransaction.h         |  3 +-
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index 53b732c2bf..916facaefc 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -1191,7 +1191,14 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
   : dbState(new DatabaseState(clusterFilePath, versionMonitorDb)) {
 	dbState->db = db;
 	dbState->dbVar->set(db);
-
+	auto stateMapKey = std::make_pair(clusterFilePath, dbState->dbProtocolVersion.get());
+	if (api->clusterSharedStateMap.find(stateMapKey) == api->clusterSharedStateMap.end() ||
+	    api->clusterSharedStateMap[stateMapKey] == nullptr) {
+		DatabaseSharedState* p = db->createSharedState();
+		api->clusterSharedStateMap[stateMapKey] = p;
+	} else {
+		db->setSharedState(api->clusterSharedStateMap[stateMapKey]);
+	}
 	if (openConnectors) {
 		if (!api->localClientDisabled) {
 			dbState->addClient(api->getLocalClient());
@@ -1531,6 +1538,15 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 		}
 	}
 
+	auto stateMapKey = std::make_pair(clusterFilePath, dbProtocolVersion.get());
+	if (MultiVersionApi::api->clusterSharedStateMap.find(stateMapKey) ==
+	        MultiVersionApi::api->clusterSharedStateMap.end() ||
+	    MultiVersionApi::api->clusterSharedStateMap[stateMapKey] == nullptr) {
+		DatabaseSharedState* p = db->createSharedState();
+		MultiVersionApi::api->clusterSharedStateMap[stateMapKey] = p;
+	} else {
+		db->setSharedState(MultiVersionApi::api->clusterSharedStateMap[stateMapKey]);
+	}
 	dbVar->set(db);
 
 	ASSERT(protocolVersionMonitor.isValid());
@@ -2147,13 +2163,6 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 		lock.leave();
 
 		Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
-		if (clusterSharedStateMap.find(clusterFile) == clusterSharedStateMap.end() ||
-		    clusterSharedStateMap[clusterFile] == nullptr) {
-			DatabaseSharedState* p = localDb->createSharedState();
-			clusterSharedStateMap[clusterFile] = p;
-		} else {
-			localDb->setSharedState(clusterSharedStateMap[clusterFile]);
-		}
 		return Reference<IDatabase>(
 		    new MultiVersionDatabase(this, threadIdx, clusterFile, Reference<IDatabase>(), localDb));
 	}
@@ -2163,13 +2172,6 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char* clusterFilePath
 	ASSERT_LE(threadCount, 1);
 
 	Reference<IDatabase> localDb = localClient->api->createDatabase(clusterFilePath);
-	if (clusterSharedStateMap.find(clusterFile) == clusterSharedStateMap.end() ||
-	    clusterSharedStateMap[clusterFile] == nullptr) {
-		DatabaseSharedState* p = localDb->createSharedState();
-		clusterSharedStateMap[clusterFile] = p;
-	} else {
-		localDb->setSharedState(clusterSharedStateMap[clusterFile]);
-	}
 	if (bypassMultiClientApi) {
 		return localDb;
 	} else {
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 9896bff0f1..79c13b104f 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -718,6 +718,8 @@ public:
 
 	bool callbackOnMainThread;
 	bool localClientDisabled;
+	// Map of (clusterFilePath + protocolVersion) -> DatabaseSharedState pointer
+	std::map<std::pair<std::string, ProtocolVersion>, DatabaseSharedState*> clusterSharedStateMap;
 
 	static bool apiVersionAtLeast(int minVersion);
 
@@ -741,7 +743,6 @@ private:
 	Reference<ClientInfo> localClient;
 	std::map<std::string, ClientDesc> externalClientDescriptions;
 	std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
-	std::map<std::string, DatabaseSharedState*> clusterSharedStateMap;
 
 	bool networkStartSetup;
 	volatile bool networkSetup;

From d8e7fea421027293d1acccd9f05bd92669be712d Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 2 Feb 2022 14:03:32 -0500
Subject: [PATCH 043/210] clean up some comments and debug changes

---
 fdbclient/ClientKnobs.h             |  1 -
 fdbclient/DatabaseContext.h         |  1 -
 fdbclient/NativeAPI.actor.cpp       |  1 -
 fdbserver/GrvProxyServer.actor.cpp  |  2 +-
 fdbserver/workloads/Cycle.actor.cpp |  1 -
 flow/Trace.cpp                      | 20 ++++++++++----------
 flow/Trace.h                        |  2 +-
 7 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index b60c902197..9cf51106b0 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -124,7 +124,6 @@ public:
 	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
 	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
-	// should these be server knobs?
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
 	int GRV_THROTTLING_THRESHOLD; // Number of iterations spent in the request queue loop which is considered throttling
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 88d4f6ccf1..e64a9605d2 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -33,7 +33,6 @@
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
-#include "fdbclient/Notified.h"
 #include "fdbrpc/QueueModel.h"
 #include "fdbrpc/MultiInterface.h"
 #include "flow/TDMetric.actor.h"
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 0d503f82a8..486a0b2b6f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5799,7 +5799,6 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		break;
 
 	case FDBTransactionOptions::USE_GRV_CACHE:
-		// Needs to be a no-op if ratekeeper is throttling
 		validateOptionValueNotPresent(value);
 		trState->options.useGrvCache = true;
 		break;
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index c198917e6c..150eef0d00 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -677,7 +677,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 				stats->lastTxnThrottled = true;
 				stats->throttleStartTime = now();
 			}
-		} else { // reply.timeThrottled <= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD
+		} else {
 			TraceEvent("DebugGrvProxyNotThrottled");
 			stats->lastTxnThrottled = false;
 		}
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp
index d4de002a9e..87c477eab4 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.actor.cpp
@@ -125,7 +125,6 @@ struct CycleWorkload : TestWorkload {
 						int r4 = self->fromValue(v3.get());
 
 						tr.clear(self->key(r)); //< Shouldn't have an effect, but will break with wrong ordering
-						tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 						tr.set(self->key(r), self->value(r3));
 						tr.set(self->key(r2), self->value(r4));
 						tr.set(self->key(r3), self->value(r2));
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index ab6e16de44..7056f00042 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -911,16 +911,16 @@ bool TraceEvent::init() {
 	enabled = enabled && (!g_network || severity >= FLOW_KNOBS->MIN_TRACE_SEVERITY);
 
 	// Backstop to throttle very spammy trace events
-	// if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
-	// 	if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
-	// 		enabled = false;
-	// 		TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
-	// 		    .suppressFor(5);
-	// 	} else {
-	// 		traceEventThrottlerCache->addAndExpire(
-	// 		    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
-	// 	}
-	// }
+	if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
+		if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
+			enabled = false;
+			TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
+			    .suppressFor(5);
+		} else {
+			traceEventThrottlerCache->addAndExpire(
+			    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
+		}
+	}
 
 	if (enabled) {
 		tmpEventMetric = std::make_unique<DynamicEventMetric>(MetricNameRef());
diff --git a/flow/Trace.h b/flow/Trace.h
index 58e58be983..52dc94aab7 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -33,7 +33,7 @@
 #include "flow/Error.h"
 #include "flow/ITrace.h"
 
-#define TRACE_DEFAULT_ROLL_SIZE (10 << 24)
+#define TRACE_DEFAULT_ROLL_SIZE (10 << 20)
 #define TRACE_DEFAULT_MAX_LOGS_SIZE (10 * TRACE_DEFAULT_ROLL_SIZE)
 
 inline int fastrand() {

From 6d05ecffc21d8497b4f753d11b17beb6102f32ab Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 2 Feb 2022 14:39:17 -0500
Subject: [PATCH 044/210] remove all multi-version client code related to grv
 cache

---
 bindings/c/CMakeLists.txt                   |  2 -
 bindings/c/fdb_c.cpp                        |  9 ----
 bindings/c/foundationdb/fdb_c.h             | 10 +++-
 bindings/c/foundationdb/fdb_c_internal.h    | 45 ----------------
 bindings/c/foundationdb/fdb_c_types.h       | 45 ----------------
 fdbclient/DatabaseContext.h                 |  5 --
 fdbclient/FDBTypes.h                        | 16 ------
 fdbclient/IClientApi.h                      |  4 --
 fdbclient/MultiVersionTransaction.actor.cpp | 44 ---------------
 fdbclient/MultiVersionTransaction.h         | 10 ----
 fdbclient/NativeAPI.actor.cpp               | 59 ++-------------------
 fdbclient/ThreadSafeTransaction.cpp         |  8 ---
 fdbclient/ThreadSafeTransaction.h           |  3 --
 13 files changed, 13 insertions(+), 247 deletions(-)
 delete mode 100644 bindings/c/foundationdb/fdb_c_internal.h
 delete mode 100644 bindings/c/foundationdb/fdb_c_types.h

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index c84f46b33c..d08bb48344 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -1,8 +1,6 @@
 set(FDB_C_SRCS
   fdb_c.cpp
   foundationdb/fdb_c.h
-  foundationdb/fdb_c_internal.h
-  foundationdb/fdb_c_types.h
   ThreadCleanup.cpp)
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index 9ca6277dd2..612f454125 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -26,7 +26,6 @@
 #include "fdbclient/MultiVersionTransaction.h"
 #include "fdbclient/MultiVersionAssignmentVars.h"
 #include "foundationdb/fdb_c.h"
-#include "foundationdb/fdb_c_internal.h"
 
 int g_api_version = 0;
 
@@ -406,14 +405,6 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
 	                        .extractPtr());
 }
 
-extern "C" DLLEXPORT DatabaseSharedState* fdb_database_create_shared_state(FDBDatabase* db) {
-	return (DatabaseSharedState*)(DB(db)->createSharedState());
-}
-
-extern "C" DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p) {
-	(DB(db)->setSharedState(p));
-}
-
 // Get network thread busyness (updated every 1s)
 // A value of 0 indicates that the client is more or less idle
 // A value of 1 (or more) indicates that the client is saturated
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index e529bfcfe8..9779582b75 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -58,12 +58,20 @@
 #include <stdint.h>
 
 #include "fdb_c_options.g.h"
-#include "fdb_c_types.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/* Pointers to these opaque types represent objects in the FDB API */
+typedef struct FDB_future FDBFuture;
+typedef struct FDB_result FDBResult;
+typedef struct FDB_database FDBDatabase;
+typedef struct FDB_transaction FDBTransaction;
+
+typedef int fdb_error_t;
+typedef int fdb_bool_t;
+
 DLLEXPORT const char* fdb_get_error(fdb_error_t code);
 
 DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);
diff --git a/bindings/c/foundationdb/fdb_c_internal.h b/bindings/c/foundationdb/fdb_c_internal.h
deleted file mode 100644
index 15312222ce..0000000000
--- a/bindings/c/foundationdb/fdb_c_internal.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * fdb_c_internal.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDB_C_INTERNAL_H
-#define FDB_C_INTERNAL_H
-#pragma once
-
-#ifndef DLLEXPORT
-#define DLLEXPORT
-#endif
-
-#include "fdb_c_types.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// forward declaration and typedef
-typedef struct DatabaseSharedState DatabaseSharedState;
-
-DLLEXPORT DatabaseSharedState* fdb_database_create_shared_state(FDBDatabase* db);
-
-DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p);
-
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/bindings/c/foundationdb/fdb_c_types.h b/bindings/c/foundationdb/fdb_c_types.h
deleted file mode 100644
index 64a7771d41..0000000000
--- a/bindings/c/foundationdb/fdb_c_types.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * fdb_c_types.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDB_C_TYPES_H
-#define FDB_C_TYPES_H
-#pragma once
-
-#ifndef DLLEXPORT
-#define DLLEXPORT
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Pointers to these opaque types represent objects in the FDB API */
-typedef struct FDB_future FDBFuture;
-typedef struct FDB_result FDBResult;
-typedef struct FDB_database FDBDatabase;
-typedef struct FDB_transaction FDBTransaction;
-
-typedef int fdb_error_t;
-typedef int fdb_bool_t;
-
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index e64a9605d2..bbd1d5fd6a 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -468,11 +468,6 @@ public:
 	int outstandingWatches;
 	int maxOutstandingWatches;
 
-	// Manage any shared state that may be used by MVC
-	DatabaseSharedState* sharedStatePtr;
-	DatabaseSharedState* initSharedState();
-	void setSharedState(DatabaseSharedState* p);
-
 	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
 	double lastTimedGrv;
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 59ecc3f072..f0299c41a4 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1171,22 +1171,6 @@ struct StorageMigrationType {
 	uint32_t type;
 };
 
-struct GRVCacheSpace {
-	Version cachedRv;
-	double lastTimedGrv;
-
-	GRVCacheSpace() : cachedRv(Version(0)), lastTimedGrv(0.0) {}
-};
-
-// This structure can be extended in the future to include additional features that required a shared state
-struct DatabaseSharedState {
-	Mutex mutexLock;
-	GRVCacheSpace grvCacheSpace;
-	int refCount;
-
-	DatabaseSharedState() : mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()) {}
-};
-
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
 	int pos = locality.find(':');
 	// locality should be either 0 or in the format '<non_empty_string>:<non_empty_string>'
diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h
index aa9b24576c..f62d588591 100644
--- a/fdbclient/IClientApi.h
+++ b/fdbclient/IClientApi.h
@@ -137,10 +137,6 @@ public:
 	// Management API, create snapshot
 	virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
 
-	// Interface to manage shared state across multiple connections to the same Database
-	virtual DatabaseSharedState* createSharedState() = 0;
-	virtual void setSharedState(DatabaseSharedState* p) = 0;
-
 	// used in template functions as the Transaction type that can be created through createTransaction()
 	using TransactionT = ITransaction;
 };
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index 916facaefc..e24e2a565a 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -446,19 +446,6 @@ ThreadFuture<Void> DLDatabase::createSnapshot(const StringRef& uid, const String
 	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
 }
 
-DatabaseSharedState* DLDatabase::createSharedState() {
-	if (!api->databaseCreateSharedState) {
-		return nullptr;
-	}
-	return api->databaseCreateSharedState(db);
-}
-
-void DLDatabase::setSharedState(DatabaseSharedState* p) {
-	if (api->databaseSetSharedState) {
-		api->databaseSetSharedState(db, p);
-	}
-}
-
 // Get network thread busyness
 double DLDatabase::getMainThreadBusyness() {
 	if (api->databaseGetMainThreadBusyness != nullptr) {
@@ -736,7 +723,6 @@ Reference<IDatabase> DLApi::createDatabase609(const char* clusterFilePath) {
 Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath) {
 	if (headerVersion >= 610) {
 		FdbCApi::FDBDatabase* db;
-		// can the FdbCApi wrapper signature be changed to add this ptr?
 		throwIfError(api->createDatabase(clusterFilePath, &db));
 		return Reference<IDatabase>(new DLDatabase(api, db));
 	} else {
@@ -1191,14 +1177,6 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
   : dbState(new DatabaseState(clusterFilePath, versionMonitorDb)) {
 	dbState->db = db;
 	dbState->dbVar->set(db);
-	auto stateMapKey = std::make_pair(clusterFilePath, dbState->dbProtocolVersion.get());
-	if (api->clusterSharedStateMap.find(stateMapKey) == api->clusterSharedStateMap.end() ||
-	    api->clusterSharedStateMap[stateMapKey] == nullptr) {
-		DatabaseSharedState* p = db->createSharedState();
-		api->clusterSharedStateMap[stateMapKey] = p;
-	} else {
-		db->setSharedState(api->clusterSharedStateMap[stateMapKey]);
-	}
 	if (openConnectors) {
 		if (!api->localClientDisabled) {
 			dbState->addClient(api->getLocalClient());
@@ -1308,19 +1286,6 @@ ThreadFuture<Void> MultiVersionDatabase::createSnapshot(const StringRef& uid, co
 	return abortableFuture(f, dbState->dbVar->get().onChange);
 }
 
-DatabaseSharedState* MultiVersionDatabase::createSharedState() {
-	if (dbState->db) {
-		return dbState->db->createSharedState();
-	}
-	return nullptr;
-}
-
-void MultiVersionDatabase::setSharedState(DatabaseSharedState* p) {
-	if (dbState->db) {
-		dbState->db->setSharedState(p);
-	}
-}
-
 // Get network thread busyness
 // Return the busyness for the main thread. When using external clients, take the larger of the local client
 // and the external client's busyness.
@@ -1538,15 +1503,6 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 		}
 	}
 
-	auto stateMapKey = std::make_pair(clusterFilePath, dbProtocolVersion.get());
-	if (MultiVersionApi::api->clusterSharedStateMap.find(stateMapKey) ==
-	        MultiVersionApi::api->clusterSharedStateMap.end() ||
-	    MultiVersionApi::api->clusterSharedStateMap[stateMapKey] == nullptr) {
-		DatabaseSharedState* p = db->createSharedState();
-		MultiVersionApi::api->clusterSharedStateMap[stateMapKey] = p;
-	} else {
-		db->setSharedState(MultiVersionApi::api->clusterSharedStateMap[stateMapKey]);
-	}
 	dbVar->set(db);
 
 	ASSERT(protocolVersionMonitor.isValid());
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 79c13b104f..7a9ebd6caf 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -108,8 +108,6 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                     int uidLength,
 	                                     uint8_t const* snapshotCommmand,
 	                                     int snapshotCommandLength);
-	DatabaseSharedState* (*databaseCreateSharedState)(FDBDatabase* database);
-	void (*databaseSetSharedState)(FDBDatabase* database, DatabaseSharedState* p);
 
 	double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
 	FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
@@ -362,9 +360,6 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
-	DatabaseSharedState* createSharedState() override;
-	void setSharedState(DatabaseSharedState* p) override;
-
 private:
 	const Reference<FdbCApi> api;
 	FdbCApi::FDBDatabase*
@@ -593,9 +588,6 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
-	DatabaseSharedState* createSharedState() override;
-	void setSharedState(DatabaseSharedState* p) override;
-
 	// private:
 
 	struct LegacyVersionMonitor;
@@ -718,8 +710,6 @@ public:
 
 	bool callbackOnMainThread;
 	bool localClientDisabled;
-	// Map of (clusterFilePath + protocolVersion) -> DatabaseSharedState pointer
-	std::map<std::pair<std::string, ProtocolVersion>, DatabaseSharedState*> clusterSharedStateMap;
 
 	static bool apiVersionAtLeast(int minVersion);
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 486a0b2b6f..60225017c9 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -209,23 +209,8 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-void updateCachedRVShared(double t, Version v, DatabaseSharedState* p) {
-	MutexHolder mutex(p->mutexLock);
-	TraceEvent("CheckpointCacheUpdateShared")
-	    .detail("Version", v)
-	    .detail("CurTime", t)
-	    .detail("LastVersion", p->grvCacheSpace.cachedRv)
-	    .detail("LastTime", p->grvCacheSpace.lastTimedGrv);
-	p->grvCacheSpace.cachedRv = v;
-	if (t > p->grvCacheSpace.lastTimedGrv) {
-		p->grvCacheSpace.lastTimedGrv = t;
-	}
-}
 
 void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (sharedStatePtr) {
-		return updateCachedRVShared(t, v, sharedStatePtr);
-	}
 	if (v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")
 		    .detail("Version", v)
@@ -244,18 +229,10 @@ void DatabaseContext::updateCachedRV(double t, Version v) {
 }
 
 Version DatabaseContext::getCachedRV() {
-	if (sharedStatePtr) {
-		MutexHolder mutex(sharedStatePtr->mutexLock);
-		return sharedStatePtr->grvCacheSpace.cachedRv;
-	}
 	return cachedRv;
 }
 
 double DatabaseContext::getLastTimedGRV() {
-	if (sharedStatePtr) {
-		MutexHolder mutex(sharedStatePtr->mutexLock);
-		return sharedStatePtr->grvCacheSpace.lastTimedGrv;
-	}
 	return lastTimedGrv;
 }
 
@@ -1335,10 +1312,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), outstandingWatches(0), sharedStatePtr(nullptr), lastTimedGrv(0.0), cachedRv(0),
-    lastTimedRkThrottle(0.0), lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID),
-    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
-    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0),
+    lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
+    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
+    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)),
     connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {
@@ -1627,13 +1604,6 @@ DatabaseContext::~DatabaseContext() {
 	if (grvUpdateHandler.isValid()) {
 		grvUpdateHandler.cancel();
 	}
-	if (sharedStatePtr) {
-		sharedStatePtr->refCount--;
-		if (sharedStatePtr->refCount <= 0) {
-			delete sharedStatePtr;
-			sharedStatePtr = nullptr;
-		}
-	}
 	for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it))
 		it->second->notifyContextDestroyed();
 	ASSERT_ABORT(server_interf.empty());
@@ -6043,16 +6013,6 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	return rep.version;
 }
 
-// ACTOR Future<Version> getDBCachedReadVersion(DatabaseContext* cx, double requestTime) {
-// 	if (requestTime - cx->lastTimedGrv.get() > CLIENT_KNOBS->MAX_VERSION_CACHE_LAG) {
-// 		wait(cx->lastTimedGrv.whenAtLeast(requestTime - CLIENT_KNOBS->MAX_VERSION_CACHE_LAG));
-// 	}
-// 	// Want to check that the cached version is at most
-// 	// MAX_VERSION_CACHE_LAG (100ms) old compared to requestTime.
-// 	ASSERT(!debug_checkVersionTime(cx->cachedRv, requestTime, "CheckStaleness"));
-// 	return cx->cachedRv;
-// }
-
 bool rkThrottlingCooledDown(DatabaseContext* cx) {
 	if (cx->lastTimedRkThrottle == 0.0) {
 		return true;
@@ -7169,17 +7129,6 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
 	return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
 }
 
-DatabaseSharedState* DatabaseContext::initSharedState() {
-	DatabaseSharedState* newState = new DatabaseSharedState();
-	setSharedState(newState);
-	return newState;
-}
-
-void DatabaseContext::setSharedState(DatabaseSharedState* p) {
-	sharedStatePtr = p;
-	sharedStatePtr->refCount++;
-}
-
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
 	state Promise<Void> destroyed = self->destroyed;
 	loop {
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index ad6032e454..eb50b0f16f 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -96,14 +96,6 @@ ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons
 	return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
 }
 
-DatabaseSharedState* ThreadSafeDatabase::createSharedState() {
-	return db->initSharedState();
-}
-
-void ThreadSafeDatabase::setSharedState(DatabaseSharedState* p) {
-	db->setSharedState(p);
-}
-
 // Return the main network thread busyness
 double ThreadSafeDatabase::getMainThreadBusyness() {
 	ASSERT(g_network);
diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h
index b93e9483a5..f61e0de2e8 100644
--- a/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/ThreadSafeTransaction.h
@@ -57,9 +57,6 @@ public:
 	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 	ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
 
-	DatabaseSharedState* createSharedState() override;
-	void setSharedState(DatabaseSharedState* p) override;
-
 private:
 	friend class ThreadSafeTransaction;
 	bool isConfigDB{ false };

From ec2bbf0343a6579d2e241a712e0ed4fc7eef69fa Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 7 Feb 2022 14:50:04 -0500
Subject: [PATCH 045/210] clean up some more trace lines and leftover code
 snippets

---
 fdbclient/ClientKnobs.cpp                    |  2 +-
 fdbclient/MultiVersionTransaction.actor.cpp  |  1 +
 fdbclient/MultiVersionTransaction.h          |  1 -
 fdbclient/NativeAPI.actor.cpp                | 28 +++-----------------
 fdbserver/GrvProxyServer.actor.cpp           |  7 +----
 fdbserver/workloads/SidebandSingle.actor.cpp |  2 +-
 6 files changed, 8 insertions(+), 33 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 834eb1e50e..cdbc6ad16e 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -123,7 +123,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( FORCE_GRV_CACHE_OFF,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
 	init( GRV_THROTTLING_THRESHOLD,                   2 );
-	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       1.0 );
+	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       0.1 );
 
 	// TaskBucket
 	init( TASKBUCKET_LOGGING_DELAY,                5.0 );
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index e24e2a565a..fa451966c9 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -1177,6 +1177,7 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
   : dbState(new DatabaseState(clusterFilePath, versionMonitorDb)) {
 	dbState->db = db;
 	dbState->dbVar->set(db);
+
 	if (openConnectors) {
 		if (!api->localClientDisabled) {
 			dbState->addClient(api->getLocalClient());
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index 7a9ebd6caf..f8bde8c000 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -108,7 +108,6 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                     int uidLength,
 	                                     uint8_t const* snapshotCommmand,
 	                                     int snapshotCommandLength);
-
 	double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
 	FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 60225017c9..e05593a701 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -31,7 +31,6 @@
 #include "contrib/fmt-8.0.1/include/fmt/format.h"
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/Knobs.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/MultiInterface.h"
 
@@ -65,14 +64,12 @@
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/sim_validation.h"
-#include "fdbserver/Knobs.h"
 #include "flow/Arena.h"
 #include "flow/ActorCollection.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
 #include "flow/IRandom.h"
-#include "flow/Trace.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Knobs.h"
@@ -1038,20 +1035,13 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state Transaction tr;
 	state double grvDelay = 0.001;
-	TraceEvent("BackgroundGrvUpdaterStart").detail("DBID", cx->dbId).detail("GrvDelay", grvDelay);
-	TraceEvent("DebugGrvCheckKnobValues")
-	    .detail("MinTraceSeverity", FLOW_KNOBS->MIN_TRACE_SEVERITY)
-	    .detail("GrvSustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD)
-	    .detail("GrvCacheRkCooldown", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN)
-	    .detail("MaxVersionCacheLag", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG)
-	    .detail("DebugUseGrvCacheChance", CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE);
 	try {
 		loop {
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
 			state double lastTime = cx->getLastTimedGRV();
 			state double lastProxyTime = cx->lastProxyRequest;
-			TraceEvent("BackgroundGrvUpdaterBefore")
+			TraceEvent(SevDebug, "BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
 			    .detail("GrvDelay", grvDelay)
@@ -1066,12 +1056,12 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					wait(success(tr.getReadVersion()));
 					cx->lastProxyRequest = curTime;
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
-					TraceEvent("BackgroundGrvUpdaterSuccess")
+					TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
 					    .detail("CachedRv", cx->getCachedRV())
 					    .detail("CachedTime", cx->getLastTimedGRV());
 				} catch (Error& e) {
-					TraceEvent("BackgroundGrvUpdaterTxnError").error(e, true);
+					TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").error(e, true);
 					wait(tr.onError(e));
 				}
 			} else {
@@ -1079,7 +1069,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent("BackgroundGrvUpdaterFailed").error(e, true);
+		TraceEvent(SevInfo, "BackgroundGrvUpdaterFailed").error(e, true);
 		throw;
 	}
 }
@@ -5950,11 +5940,7 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	double latency = now() - trState->startTime;
 	trState->cx->lastProxyRequest = trState->startTime;
 	trState->cx->updateCachedRV(trState->startTime, rep.version);
-	TraceEvent("DebugGrvQueueIterations").detail("TimeThrottled", rep.queueIterations);
 	if (rep.rkThrottled && trState->options.priority != TransactionPriority::IMMEDIATE) {
-		TraceEvent("DebugGrvThrottleObserved")
-		    .detail("TimeThrottled", rep.queueIterations)
-		    .detail("Threshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD);
 		trState->cx->lastTimedRkThrottle = now();
 	}
 	trState->cx->GRVLatencies.addSample(latency);
@@ -6017,10 +6003,6 @@ bool rkThrottlingCooledDown(DatabaseContext* cx) {
 	if (cx->lastTimedRkThrottle == 0.0) {
 		return true;
 	}
-	TraceEvent("DebugGrvRkCd")
-	    .detail("LastTimedRkThrottle", cx->lastTimedRkThrottle)
-	    .detail("TimeElapsed", now() - cx->lastTimedRkThrottle)
-	    .detail("CooldownTime", CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 }
 
@@ -6030,7 +6012,6 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE ||
 		     trState->options.useGrvCache) &&
 		    rkThrottlingCooledDown(getDatabase().getPtr())) {
-			TraceEvent("DebugGrvEnterCachePath");
 			// Upon our first request to use cached RVs, start the background updater
 			if (!trState->cx->grvUpdateHandler.isValid()) {
 				trState->cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
@@ -6039,7 +6020,6 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			double lastTime = trState->cx->getLastTimedGRV();
 			double requestTime = now();
 			if (requestTime - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
-				TraceEvent("DebugGrvUseCache").detail("LastRV", rv).detail("LastTime", format("%.6f", lastTime));
 				ASSERT(!debug_checkVersionTime(rv, requestTime, "CheckStaleness"));
 				readVersion = rv;
 				return readVersion;
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 150eef0d00..7176e72761 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -654,7 +654,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		}
 
 		reply.queueIterations = request.queueIterations;
-		TraceEvent("DebugGrvProxyThrottleCheck")
+		TraceEvent(SevDebug, "DebugGrvProxyThrottleCheck")
 		    .detail("QueueIterations", reply.queueIterations)
 		    .detail("ThrottleThreshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD)
 		    .detail("LastTxnThrottled", stats->lastTxnThrottled)
@@ -662,23 +662,18 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		    .detail("Diff", now() - stats->throttleStartTime)
 		    .detail("SustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
 		if (reply.queueIterations >= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
-			TraceEvent("DebugGrvProxyThrottled");
 			if (stats->lastTxnThrottled) {
-				TraceEvent("DebugGrvProxyLastTxnThrottled");
 				// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 				if (now() - stats->throttleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
-					TraceEvent("DebugGrvProxyRkThrottled");
 					reply.rkThrottled = true;
 				}
 			} else { // !stats->lastTxnThrottled
-				TraceEvent("DebugGrvProxyLastTxnNotThrottled");
 				// If not previously throttled, this request/reply is our new starting point
 				// for judging whether we are being actively throttled by ratekeeper now
 				stats->lastTxnThrottled = true;
 				stats->throttleStartTime = now();
 			}
 		} else {
-			TraceEvent("DebugGrvProxyNotThrottled");
 			stats->lastTxnThrottled = false;
 		}
 		request.reply.send(reply);
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index 4c4c40ad9e..c25476c88d 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

From 7e5ad88780fc0fe65394dac2682ad6cf68356bbd Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 7 Feb 2022 15:31:59 -0500
Subject: [PATCH 046/210] formatting for Github CI

---
 fdbclient/NativeAPI.actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index c208a3fe1c..c896301df4 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -206,7 +206,6 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-
 void DatabaseContext::updateCachedRV(double t, Version v) {
 	if (v >= cachedRv) {
 		TraceEvent("CheckpointCacheUpdate")

From 7d17e000038bdda2a7fc6bb6a851cdfe15274f8c Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 7 Feb 2022 16:58:29 -0500
Subject: [PATCH 047/210] clean up extra trace messages in SidebandSingle
 workload

---
 fdbserver/workloads/SidebandSingle.actor.cpp | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index c25476c88d..f51109104c 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -170,14 +170,11 @@ struct SidebandSingleWorkload : TestWorkload {
 				try {
 					// tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
 					tr.set(messageKey, LiteralStringRef("deadbeef"));
-					TraceEvent("DebugSidebandBeforeCommit");
 					wait(tr.commit());
 					commitVersion = tr.getCommittedVersion();
-					TraceEvent("DebugSidebandAfterCommit").detail("CommitVersion", commitVersion);
 					break;
 				} catch (Error& e) {
 					if (e.code() == error_code_commit_unknown_result) {
-						TraceEvent("DebugSidebandUnknownResult");
 						unknown = true;
 						++self->messages;
 						checker.updates.send(SidebandMessage(key, invalidVersion));
@@ -202,10 +199,8 @@ struct SidebandSingleWorkload : TestWorkload {
 			state Transaction tr(cx);
 			loop {
 				try {
-					TraceEvent("DebugSidebandCacheGetBefore");
 					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 					state Optional<Value> val = wait(tr.get(messageKey));
-					TraceEvent("DebugSidebandCacheGetAfter");
 					if (!val.present()) {
 						TraceEvent(SevError, "CausalConsistencyError1", self->interf.id())
 						    .detail("MessageKey", messageKey.toString().c_str())
@@ -214,15 +209,12 @@ struct SidebandSingleWorkload : TestWorkload {
 						            tr.getReadVersion().get()); // will assert that ReadVersion is set
 						++self->consistencyErrors;
 					} else if (val.get() != LiteralStringRef("deadbeef")) {
-						TraceEvent("DebugSidebandOldBeef");
 						// check again without cache, and if it's the same, that's expected
 						state Transaction tr2(cx);
 						state Optional<Value> val2;
 						loop {
 							try {
-								TraceEvent("DebugSidebandNoCacheGetBefore");
 								wait(store(val2, tr2.get(messageKey)));
-								TraceEvent("DebugSidebandNoCacheGetAfter");
 								break;
 							} catch (Error& e) {
 								TraceEvent("DebugSidebandNoCacheError").error(e, true);

From 458e70827245890fb2799a5173665ecc7ecdc1c5 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 10 Feb 2022 16:17:54 -0500
Subject: [PATCH 048/210] addressed code review comments: renamed variables,
 small functional changes, style changes

---
 fdbclient/CommitProxyInterface.h             |  2 +-
 fdbclient/DatabaseContext.h                  | 14 ++--
 fdbclient/NativeAPI.actor.cpp                | 68 ++++++++++----------
 fdbclient/ReadYourWrites.actor.cpp           |  6 --
 fdbclient/vexillographer/fdb.options         |  4 +-
 fdbrpc/sim_validation.cpp                    | 12 ++--
 fdbserver/GrvProxyServer.actor.cpp           |  6 +-
 fdbserver/workloads/SidebandSingle.actor.cpp | 10 ++-
 8 files changed, 64 insertions(+), 58 deletions(-)

diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index cc86294d6f..a0d396694c 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -199,7 +199,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
 	uint32_t queueIterations = 0;
-	bool rkThrottled;
+	bool rkThrottled = false;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index bbd1d5fd6a..0ba157a04d 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -470,16 +470,16 @@ public:
 
 	// GRV Cache
 	// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
-	double lastTimedGrv;
-	Version cachedRv;
-	void updateCachedRV(double t, Version v);
-	Version getCachedRV();
-	double getLastTimedGRV();
-	double lastTimedRkThrottle;
+	double lastGrvTime;
+	Version cachedReadVersion;
+	void updateCachedReadVersion(double t, Version v);
+	Version getCachedReadVersion();
+	double getLastGrvTime();
+	double lastRkThrottleTime;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
 	// we want to track the last time in order to periodically contact the proxy to check for throttling
-	double lastProxyRequest;
+	double lastProxyRequestTime;
 
 	int snapshotRywEnabled;
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index c896301df4..4c8947d81f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -206,30 +206,30 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
 	}
 }
 
-void DatabaseContext::updateCachedRV(double t, Version v) {
-	if (v >= cachedRv) {
-		TraceEvent("CheckpointCacheUpdate")
+void DatabaseContext::updateCachedReadVersion(double t, Version v) {
+	if (v >= cachedReadVersion) {
+		TraceEvent(SevDebug, "CachedReadVersionUpdate")
 		    .detail("Version", v)
-		    .detail("CurTime", t)
-		    .detail("LastVersion", cachedRv)
-		    .detail("LastTime", lastTimedGrv);
-		cachedRv = v;
+		    .detail("GrvStartTime", t)
+		    .detail("LastVersion", cachedReadVersion)
+		    .detail("LastTime", lastGrvTime);
+		cachedReadVersion = v;
 		// Since the time is based on the start of the request, it's possible that we
 		// get a newer version with an older time.
 		// (Request started earlier, but was latest to reach the proxy)
 		// Only update time when strictly increasing (?)
-		if (t > lastTimedGrv) {
-			lastTimedGrv = t;
+		if (t > lastGrvTime) {
+			lastGrvTime = t;
 		}
 	}
 }
 
-Version DatabaseContext::getCachedRV() {
-	return cachedRv;
+Version DatabaseContext::getCachedReadVersion() {
+	return cachedReadVersion;
 }
 
-double DatabaseContext::getLastTimedGRV() {
-	return lastTimedGrv;
+double DatabaseContext::getLastGrvTime() {
+	return lastGrvTime;
 }
 
 Reference<StorageServerInfo> StorageServerInfo::getInterface(DatabaseContext* cx,
@@ -1036,16 +1036,18 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state double grvDelay = 0.001;
 	try {
 		loop {
+			if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF)
+				return Void();
 			wait(refreshTransaction(cx, &tr));
 			state double curTime = now();
-			state double lastTime = cx->getLastTimedGRV();
-			state double lastProxyTime = cx->lastProxyRequest;
+			state double lastTime = cx->getLastGrvTime();
+			state double lastProxyTime = cx->lastProxyRequestTime;
 			TraceEvent(SevDebug, "BackgroundGrvUpdaterBefore")
 			    .detail("CurTime", curTime)
 			    .detail("LastTime", lastTime)
 			    .detail("GrvDelay", grvDelay)
-			    .detail("CachedRv", cx->getCachedRV())
-			    .detail("CachedTime", cx->getLastTimedGRV())
+			    .detail("CachedReadVersion", cx->getCachedReadVersion())
+			    .detail("CachedTime", cx->getLastGrvTime())
 			    .detail("Gap", curTime - lastTime)
 			    .detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
 			if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) ||
@@ -1053,18 +1055,19 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 				try {
 					tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
 					wait(success(tr.getReadVersion()));
-					cx->lastProxyRequest = curTime;
+					cx->lastProxyRequestTime = curTime;
 					grvDelay = (grvDelay + (now() - curTime)) / 2.0;
 					TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
 					    .detail("GrvDelay", grvDelay)
-					    .detail("CachedRv", cx->getCachedRV())
-					    .detail("CachedTime", cx->getLastTimedGRV());
+					    .detail("CachedReadVersion", cx->getCachedReadVersion())
+					    .detail("CachedTime", cx->getLastGrvTime());
 				} catch (Error& e) {
 					TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").error(e, true);
 					wait(tr.onError(e));
 				}
 			} else {
-				wait(delay(0.001 + (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)));
+				wait(delay(0.001 + std::min(CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG - (curTime - lastProxyTime),
+				                            (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime))));
 			}
 		}
 	} catch (Error& e) {
@@ -1301,8 +1304,8 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), outstandingWatches(0), lastTimedGrv(0.0), cachedRv(0), lastTimedRkThrottle(0.0),
-    lastProxyRequest(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
+    bytesPerCommit(1000), outstandingWatches(0), lastGrvTime(0.0), cachedReadVersion(0), lastRkThrottleTime(0.0),
+    lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
     clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
     healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
@@ -5352,7 +5355,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
 					if (CLIENT_BUGGIFY) {
 						throw commit_unknown_result();
 					}
-					trState->cx->updateCachedRV(grvTime, v);
+					trState->cx->updateCachedReadVersion(grvTime, v);
 					if (debugID.present())
 						TraceEvent(interval.end()).detail("CommittedVersion", v);
 					trState->committedVersion = v;
@@ -5959,10 +5962,10 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	state Span span(spanContext, location, { trState->spanID });
 	GetReadVersionReply rep = wait(f);
 	double latency = now() - trState->startTime;
-	trState->cx->lastProxyRequest = trState->startTime;
-	trState->cx->updateCachedRV(trState->startTime, rep.version);
+	trState->cx->lastProxyRequestTime = trState->startTime;
+	trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
 	if (rep.rkThrottled && trState->options.priority != TransactionPriority::IMMEDIATE) {
-		trState->cx->lastTimedRkThrottle = now();
+		trState->cx->lastRkThrottleTime = now();
 	}
 	trState->cx->GRVLatencies.addSample(latency);
 	if (trState->trLogInfo)
@@ -6021,10 +6024,10 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 }
 
 bool rkThrottlingCooledDown(DatabaseContext* cx) {
-	if (cx->lastTimedRkThrottle == 0.0) {
+	if (cx->lastRkThrottleTime == 0.0) {
 		return true;
 	}
-	return (now() - cx->lastTimedRkThrottle > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
+	return (now() - cx->lastRkThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 }
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
@@ -6037,8 +6040,8 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			if (!trState->cx->grvUpdateHandler.isValid()) {
 				trState->cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
 			}
-			Version rv = trState->cx->getCachedRV();
-			double lastTime = trState->cx->getLastTimedGRV();
+			Version rv = trState->cx->getCachedReadVersion();
+			double lastTime = trState->cx->getLastGrvTime();
 			double requestTime = now();
 			if (requestTime - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
 				ASSERT(!debug_checkVersionTime(rv, requestTime, "CheckStaleness"));
@@ -6046,9 +6049,6 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 				return readVersion;
 			} // else go through regular GRV path
 		}
-		if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && trState->cx->grvUpdateHandler.isValid()) {
-			trState->cx->grvUpdateHandler.cancel();
-		}
 		++trState->cx->transactionReadVersions;
 		flags |= trState->options.getReadVersionFlags;
 		switch (trState->options.priority) {
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index fd495382b2..fdc1d5c771 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2410,12 +2410,6 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 		validateOptionValueNotPresent(value);
 		options.bypassUnreadable = true;
 		break;
-	case FDBTransactionOptions::USE_GRV_CACHE:
-		validateOptionValueNotPresent(value);
-		options.useGrvCache = true;
-	case FDBTransactionOptions::SKIP_GRV_CACHE:
-		validateOptionValueNotPresent(value);
-		options.skipGrvCache = true;
 	default:
 		break;
 	}
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index cd87ed2851..3e91511bf7 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -293,9 +293,9 @@ description is not currently required but encouraged.
     <Option name="bypass_unreadable" code="1100"
                 description="Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp." />            
     <Option name="use_grv_cache" code="1101"
-            description="Allows this transaction to use cached GRV from the database context." />
+            description="Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions." />
     <Option name="skip_grv_cache" code="1102"
-            description="Specifically instruct this transaction to NOT use cached GRV." />
+            description="Specifically instruct this transaction to NOT use cached GRV. Primarily used for the read version cache's background updater to avoid attempting to read a cached entry in specific situations." />
   </Scope>
 
   <!-- The enumeration values matter - do not change them without
diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp
index eb817b1195..d2344dbe70 100644
--- a/fdbrpc/sim_validation.cpp
+++ b/fdbrpc/sim_validation.cpp
@@ -25,7 +25,7 @@
 
 // used for simulation validations
 static std::map<std::string, int64_t> validationData;
-static std::map<int64_t, double> validationData2;
+static std::map<int64_t, double> timedVersionsValidationData;
 static std::set<UID> disabledMachines;
 
 void debug_setVersionCheckEnabled(UID uid, bool enabled) {
@@ -131,23 +131,23 @@ void debug_setCheckRelocationDuration(bool check) {
 void debug_advanceVersionTimestamp(int64_t version, double t) {
 	if (!g_network->isSimulated() || g_simulator.extraDB)
 		return;
-	validationData2[version] = t;
+	timedVersionsValidationData[version] = t;
 }
 
 bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
 	if (!g_network->isSimulated() || g_simulator.extraDB)
 		return false;
-	if (!validationData2.count(version)) {
+	if (!timedVersionsValidationData.count(version)) {
 		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
 		    .detail("VersionChecking", version)
 		    .detail("TimeChecking", t);
 		return false;
 	}
-	if (t > validationData2[version]) {
-		TraceEvent(sev, (context + "DurabilityError").c_str())
+	if (t > timedVersionsValidationData[version]) {
+		TraceEvent(sev, (context + "VersionTimeError").c_str())
 		    .detail("VersionChecking", version)
 		    .detail("TimeChecking", t)
-		    .detail("MaxTime", validationData2[version]);
+		    .detail("MaxTime", timedVersionsValidationData[version]);
 		return true;
 	}
 	return false;
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 7176e72761..a3fe28b002 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -674,7 +674,11 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 				stats->throttleStartTime = now();
 			}
 		} else {
-			stats->lastTxnThrottled = false;
+			// If an immediate priority txn comes in, it may get processed on the first time through and
+			// reset our throttling state prematurely.
+			if (request.priority != TransactionPriority::IMMEDIATE) {
+				stats->lastTxnThrottled = false;
+			}
 		}
 		request.reply.send(reply);
 		++stats->txnRequestOut;
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index f51109104c..aad4b07738 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -24,6 +24,15 @@
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+/*
+ * This workload is modelled off the Sideband workload, except it uses a single
+ * mutator and checker rather than several. In addition to ordinary consistency checks,
+ * it also checks the consistency of the cached read versions when using the
+ * USE_GRV_CACHE transaction option, specifically when commit_unknown_result
+ * produces a maybe/maybe-not written scenario. It makes sure that a cached read of an
+ * unknown result matches the regular read of that same key and is not too stale.
+ */
+
 struct SidebandMessage {
 	constexpr static FileIdentifier file_identifier = 11862047;
 	uint64_t key;
@@ -168,7 +177,6 @@ struct SidebandSingleWorkload : TestWorkload {
 			// second set, the checker should see this, no retries on unknown result
 			loop {
 				try {
-					// tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
 					tr.set(messageKey, LiteralStringRef("deadbeef"));
 					wait(tr.commit());
 					commitVersion = tr.getCommittedVersion();

From 2e63ac6963fed1a66a0924e64f544290cf4a3d87 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Thu, 10 Feb 2022 17:01:09 -0500
Subject: [PATCH 049/210] code review changes: additional test checks, cleanup
 extra code, mark internal option hidden

---
 fdbclient/CommitProxyInterface.h             |  2 +-
 fdbclient/NativeAPI.actor.cpp                |  6 ++++--
 fdbclient/ReadYourWrites.h                   |  2 --
 fdbclient/vexillographer/fdb.options         |  3 ++-
 fdbserver/workloads/SidebandSingle.actor.cpp | 10 +++++++++-
 5 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index a0d396694c..7e670ed08a 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -203,7 +203,7 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
-	GetReadVersionReply() : version(invalidVersion), locked(false), rkThrottled(false) {}
+	GetReadVersionReply() : version(invalidVersion), locked(false) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4c8947d81f..4d05bf2974 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1066,8 +1066,10 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					wait(tr.onError(e));
 				}
 			} else {
-				wait(delay(0.001 + std::min(CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG - (curTime - lastProxyTime),
-				                            (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime))));
+				wait(
+				    delay(std::max(0.001,
+				                   std::min(CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG - (curTime - lastProxyTime),
+				                            (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)))));
 			}
 		}
 	} catch (Error& e) {
diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h
index 147cc75499..b8ccd23e54 100644
--- a/fdbclient/ReadYourWrites.h
+++ b/fdbclient/ReadYourWrites.h
@@ -44,8 +44,6 @@ struct ReadYourWritesTransactionOptions {
 	int maxRetries;
 	int snapshotRywEnabled;
 	bool bypassUnreadable : 1;
-	bool useGrvCache : 1;
-	bool skipGrvCache : 1;
 
 	ReadYourWritesTransactionOptions() {}
 	explicit ReadYourWritesTransactionOptions(Transaction const& tr);
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 3e91511bf7..c4845ae8bd 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -295,7 +295,8 @@ description is not currently required but encouraged.
     <Option name="use_grv_cache" code="1101"
             description="Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions." />
     <Option name="skip_grv_cache" code="1102"
-            description="Specifically instruct this transaction to NOT use cached GRV. Primarily used for the read version cache's background updater to avoid attempting to read a cached entry in specific situations." />
+            description="Specifically instruct this transaction to NOT use cached GRV. Primarily used for the read version cache's background updater to avoid attempting to read a cached entry in specific situations."
+            hidden="true"/>
   </Scope>
 
   <!-- The enumeration values matter - do not change them without
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index aad4b07738..0ec80a9ef9 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -217,6 +217,14 @@ struct SidebandSingleWorkload : TestWorkload {
 						            tr.getReadVersion().get()); // will assert that ReadVersion is set
 						++self->consistencyErrors;
 					} else if (val.get() != LiteralStringRef("deadbeef")) {
+						// If we read something NOT "deadbeef" and there was no commit_unknown_result,
+						// the cache somehow read a stale version of our key
+						if (message.commitVersion != invalidVersion) {
+							TraceEvent(SevError, "CausalConsistencyError2", self->interf.id())
+							    .detail("MessageKey", messageKey.toString().c_str());
+							++self->consistencyErrors;
+							break;
+						}
 						// check again without cache, and if it's the same, that's expected
 						state Transaction tr2(cx);
 						state Optional<Value> val2;
@@ -230,7 +238,7 @@ struct SidebandSingleWorkload : TestWorkload {
 							}
 						}
 						if (val != val2) {
-							TraceEvent(SevError, "CausalConsistencyError2", self->interf.id())
+							TraceEvent(SevError, "CausalConsistencyError3", self->interf.id())
 							    .detail("MessageKey", messageKey.toString().c_str())
 							    .detail("Val1", val)
 							    .detail("Val2", val2)

From edfff755bf1939e757612b5073471267bcdc8230 Mon Sep 17 00:00:00 2001
From: vikasgupta8 <vikas.gupta8@ibm.com>
Date: Fri, 11 Feb 2022 06:17:15 +0000
Subject: [PATCH 050/210] added support for ppc64le

---
 bindings/c/CMakeLists.txt                     |  2 +
 bindings/c/generate_asm.py                    | 50 ++++++++++++++++++-
 bindings/java/CMakeLists.txt                  |  2 +
 .../main/com/apple/foundationdb/JNIUtil.java  |  2 +-
 cmake/ConfigureCompiler.cmake                 |  5 +-
 flow/Platform.actor.cpp                       |  4 +-
 flow/Platform.h                               | 14 +++++-
 flow/ThreadPrimitives.h                       |  4 +-
 flow/crc32c.cpp                               |  8 ++-
 9 files changed, 82 insertions(+), 9 deletions(-)

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index d08bb48344..704e7a7dc0 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -18,6 +18,8 @@ endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
   set(cpu "aarch64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+  set(cpu "ppc64le")
 endif()
 
 set(IS_ARM_MAC NO)
diff --git a/bindings/c/generate_asm.py b/bindings/c/generate_asm.py
index b0e9a0e7ad..3acd14abed 100755
--- a/bindings/c/generate_asm.py
+++ b/bindings/c/generate_asm.py
@@ -59,9 +59,10 @@ def write_windows_asm(asmfile, functions):
 
 
 def write_unix_asm(asmfile, functions, prefix):
-    if cpu != "aarch64":
+    if cpu != "aarch64" and cpu!= "ppc64le":
         asmfile.write(".intel_syntax noprefix\n")
 
+    i = 0
     if os == 'linux' or os == 'freebsd':
         asmfile.write("\n.data\n")
         for f in functions:
@@ -70,8 +71,13 @@ def write_unix_asm(asmfile, functions, prefix):
         if os == 'linux' or os == 'freebsd':
             asmfile.write("\n.text\n")
             for f in functions:
+                if cpu == "ppc64le":
+                    asmfile.write("\n.LC%d:\n" % (i))
+                    asmfile.write("\t.quad \tfdb_api_ptr_%s\n" % (f))
+                    asmfile.write("\t.align 2\n")
+                    i = i + 1
                 asmfile.write("\t.global %s\n\t.type %s, @function\n" % (f, f))
-
+    i = 0
     for f in functions:
         asmfile.write("\n.globl %s%s\n" % (prefix, f))
         if cpu == 'aarch64' and os == 'osx':
@@ -118,6 +124,46 @@ def write_unix_asm(asmfile, functions, prefix):
                 assert False, '{} not supported for Arm yet'.format(os)
             asmfile.write("\tldr x8, [x8]\n")
             asmfile.write("\tbr x8\n")
+        elif cpu == "ppc64le":
+            asmfile.write("\n.LCF%d:\n" % (i))
+            asmfile.write("\taddis 2,12,.TOC.-.LCF%d@ha\n" % (i))
+            asmfile.write("\taddi 2,2,.TOC.-.LCF%d@l\n" % (i))
+            asmfile.write("\tmflr 0\n")
+            asmfile.write("\tstd 31, -8(1)\n")
+            asmfile.write("\tstd     0,16(1)\n")
+            asmfile.write("\tstdu    1,-192(1)\n")
+            #asmfile.write("\tstd 2,24(1)\n")
+            asmfile.write("\taddis 11,2,.LC%d@toc@ha\n" % (i))
+            asmfile.write("\tld 11,.LC%d@toc@l(11)\n" % (i))
+            asmfile.write("\tld 12,0(11)\n")
+            asmfile.write("\tstd 2,24(1)\n")
+            asmfile.write("\tlwa 11,344(1)\n")
+            asmfile.write("\tmtctr 12\n")
+            asmfile.write("\tstd 11,152(1)\n")
+            asmfile.write("\tlwa 11,352(1)\n")
+            asmfile.write("\tstd 11,160(1)\n")
+            asmfile.write("\tlwa 11,336(1)\n")
+            asmfile.write("\tstd 11,144(1)\n")
+            asmfile.write("\tlwa 11,328(1)\n")
+            asmfile.write("\tstd 11,136(1)\n")
+            asmfile.write("\tlwa 11,320(1)\n")
+            asmfile.write("\tstd 11,128(1)\n")
+            asmfile.write("\tlwa 11,312(1)\n")
+            asmfile.write("\tstd 11,120(1)\n")
+            asmfile.write("\tlwa 11,304(1)\n")
+            asmfile.write("\tstd 11,112(1)\n")
+            asmfile.write("\tld 11,296(1)\n")
+            asmfile.write("\tstd 11,104(1)\n")
+            asmfile.write("\tlwa 11,288(1)\n")
+            asmfile.write("\tstd 11,96(1)\n")
+            asmfile.write("\tbctrl\n")
+            asmfile.write("\tld 2,24(1)\n")
+            asmfile.write("\taddi 1,1,192\n")
+            asmfile.write("\tld 0,16(1)\n")
+            asmfile.write("\tld 31, -8(1)\n")
+            asmfile.write("\tmtlr 0\n")
+            asmfile.write("\tblr\n")
+            i = i + 1
         else:
             asmfile.write(
                 "\tmov r11, qword ptr [%sfdb_api_ptr_%s@GOTPCREL+rip]\n" % (prefix, f))
diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt
index 40727063f6..d83305def4 100644
--- a/bindings/java/CMakeLists.txt
+++ b/bindings/java/CMakeLists.txt
@@ -228,6 +228,8 @@ if(NOT OPEN_FOR_IDE)
   else()
    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
      set(lib_destination "linux/aarch64")
+   elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+     set(lib_destination "linux/ppc64le")
    else()
      set(lib_destination "linux/amd64")
     endif()
diff --git a/bindings/java/src/main/com/apple/foundationdb/JNIUtil.java b/bindings/java/src/main/com/apple/foundationdb/JNIUtil.java
index a3811bd029..cf4e0ade06 100644
--- a/bindings/java/src/main/com/apple/foundationdb/JNIUtil.java
+++ b/bindings/java/src/main/com/apple/foundationdb/JNIUtil.java
@@ -182,7 +182,7 @@ public class JNIUtil {
 	private static OS getRunningOS() {
 		String osname = System.getProperty("os.name").toLowerCase();
 		String arch = System.getProperty("os.arch");
-		if (!arch.equals("amd64") && !arch.equals("x86_64") && !arch.equals("aarch64")) {
+		if (!arch.equals("amd64") && !arch.equals("x86_64") && !arch.equals("aarch64") && !arch.equals("ppc64le")) {
 			throw new IllegalStateException("Unknown or unsupported arch: " + arch);
 		}
 		if (osname.startsWith("windows")) {
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e2856ad0fd..d6686db885 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -219,7 +219,7 @@ else()
   endif()
   if(STATIC_LINK_LIBCXX)
     if (NOT USE_LIBCXX AND NOT APPLE)
-      add_link_options(-static-libstdc++ -static-libgcc)
+	add_link_options(-static-libstdc++ -static-libgcc)
     endif()
   endif()
   # # Instruction sets we require to be supported by the CPU
@@ -349,6 +349,9 @@ else()
     add_compile_options(-march=armv8.2-a+crc+simd)
   endif()
 
+  if (CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+      add_compile_options(-m64 -mcpu=power9 -mtune=power9 -DNO_WARN_X86_INTRINSICS)
+  endif()
   # Check whether we can use dtrace probes
   include(CheckSymbolExists)
   check_symbol_exists(DTRACE_PROBE sys/sdt.h SUPPORT_DTRACE)
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 3194bb5c34..aabba21917 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -85,7 +85,7 @@
 #include <ftw.h>
 #include <pwd.h>
 #include <sched.h>
-#ifndef __aarch64__
+#if !defined(__aarch64__) && !defined(__powerpc64__)
 #include <cpuid.h>
 #endif
 
@@ -3194,6 +3194,8 @@ bool isHwCrcSupported() {
 	return (info[2] & (1 << 20)) != 0;
 #elif defined(__aarch64__)
 	return true; /* force to use crc instructions */
+#elif defined(__powerpc64__)
+	return false; /* force not to use crc instructions */
 #elif defined(__unixish__)
 	uint32_t eax, ebx, ecx, edx, level = 1, count = 0;
 	__cpuid_count(level, count, eax, ebx, ecx, edx);
diff --git a/flow/Platform.h b/flow/Platform.h
index 75d698ea09..13c8740ef2 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -441,6 +441,8 @@ inline static uint64_t timestampCounter() {
 	asm volatile("mrs %0, cntvct_el0" : "=r"(timer));
 	return timer;
 }
+#elif defined(_powerpc64_)
+#include <emmintrin.h>
 #elif defined(__linux__)
 #include <x86intrin.h>
 #define timestampCounter() __rdtsc()
@@ -467,6 +469,14 @@ inline static uint64_t __rdtsc() {
 	return (lo | (hi << 32));
 }
 #endif
+#elif defined(__powerpc64__) || defined(__ppc64__)
+inline static uint64_t __rdtsc() {
+       uint64_t tb;
+       __asm__ volatile (\
+         "mfspr %0, 268"
+         : "=r" (tb));
+       return tb;
+}
 #endif
 
 #if defined(__linux__)
@@ -826,13 +836,15 @@ static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
 	return ret;
 }
 #else
+#ifndef __powerpc64__
 // Intel
 #define hwCrc32cU8(c, v) _mm_crc32_u8(c, v)
 #define hwCrc32cU32(c, v) _mm_crc32_u32(c, v)
 #define hwCrc32cU64(c, v) _mm_crc32_u64(c, v)
 #endif
+#endif
 
-#ifdef __aarch64__
+#if !defined(__aarch64__) && !defined(__powerpc64__)
 #define _MM_HINT_T0 0 /* dummy -- not used */
 #endif
 
diff --git a/flow/ThreadPrimitives.h b/flow/ThreadPrimitives.h
index c51be6cfee..c0913a23c9 100644
--- a/flow/ThreadPrimitives.h
+++ b/flow/ThreadPrimitives.h
@@ -64,8 +64,10 @@ public:
 	}
 	void enter() {
 		while (isLocked.test_and_set(std::memory_order_acquire))
-#ifdef __aarch64__
+#if defined(__aarch64__)
 			__asm__ volatile("isb");
+#elif defined(__powerpc64__)
+			__asm__ volatile ("or 27,27,27" ::: "memory");
 #else
 			_mm_pause();
 #endif
diff --git a/flow/crc32c.cpp b/flow/crc32c.cpp
index e9339333a5..07cfac5ede 100644
--- a/flow/crc32c.cpp
+++ b/flow/crc32c.cpp
@@ -27,7 +27,7 @@
 
 #include "flow/crc32c.h"
 
-#ifndef __aarch64__
+#if !defined(__aarch64__) && !defined(__powerpc64__)
 #include <nmmintrin.h>
 #endif
 #include <stdio.h>
@@ -133,10 +133,11 @@ static inline uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) {
 }
 
 /* Compute CRC-32C using the hardware instruction. */
-#if (defined(__clang__) || defined(__GNUG__)) && (!defined(__aarch64__))
+#if (defined(__clang__) || defined(__GNUG__) && !defined(__aarch64__) && !defined(__powerpc64__))
 /* Enable SSE CEC instructions on Intel processor */
 __attribute__((target("sse4.2")))
 #endif
+#ifndef __powerpc64__
 static uint32_t
 append_hw(uint32_t crc, const uint8_t* buf, size_t len) {
 	const uint8_t* next = buf;
@@ -263,12 +264,15 @@ append_hw(uint32_t crc, const uint8_t* buf, size_t len) {
 	/* return a post-processed crc */
 	return static_cast<uint32_t>(crc0) ^ 0xffffffff;
 }
+#endif
 
 static bool hw_available = platform::isHwCrcSupported();
 
 extern "C" uint32_t crc32c_append(uint32_t crc, const uint8_t* input, size_t length) {
+#ifndef __powerpc64__
 	if (hw_available)
 		return append_hw(crc, input, length);
 	else
+#endif
 		return append_table(crc, input, length);
 }

From 15159a5debe79f1bfbecd3920f3edae7f434a315 Mon Sep 17 00:00:00 2001
From: root <root@p006vm88.pbm.ihost.com>
Date: Fri, 11 Feb 2022 09:57:24 +0000
Subject: [PATCH 051/210] resolved format error

---
 cmake/ConfigureCompiler.cmake |  2 +-
 flow/Platform.h               | 10 +++++-----
 flow/ThreadPrimitives.h       |  6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index d6686db885..586753bdc5 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -219,7 +219,7 @@ else()
   endif()
   if(STATIC_LINK_LIBCXX)
     if (NOT USE_LIBCXX AND NOT APPLE)
-	add_link_options(-static-libstdc++ -static-libgcc)
+	    add_link_options(-static-libstdc++ -static-libgcc)
     endif()
   endif()
   # # Instruction sets we require to be supported by the CPU
diff --git a/flow/Platform.h b/flow/Platform.h
index 13c8740ef2..24e10d59f8 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -471,11 +471,11 @@ inline static uint64_t __rdtsc() {
 #endif
 #elif defined(__powerpc64__) || defined(__ppc64__)
 inline static uint64_t __rdtsc() {
-       uint64_t tb;
-       __asm__ volatile (\
-         "mfspr %0, 268"
-         : "=r" (tb));
-       return tb;
+	uint64_t tb;
+	__asm__ volatile (\
+			"mfspr %0, 268"
+			: "=r" (tb));
+	return tb;
 }
 #endif
 
diff --git a/flow/ThreadPrimitives.h b/flow/ThreadPrimitives.h
index c0913a23c9..44faa72eff 100644
--- a/flow/ThreadPrimitives.h
+++ b/flow/ThreadPrimitives.h
@@ -65,11 +65,11 @@ public:
 	void enter() {
 		while (isLocked.test_and_set(std::memory_order_acquire))
 #if defined(__aarch64__)
-			__asm__ volatile("isb");
+		__asm__ volatile("isb");
 #elif defined(__powerpc64__)
-			__asm__ volatile ("or 27,27,27" ::: "memory");
+		__asm__ volatile ("or 27,27,27" ::: "memory");
 #else
-			_mm_pause();
+		_mm_pause();
 #endif
 #if VALGRIND
 		ANNOTATE_RWLOCK_ACQUIRED(this, true);

From d61519c53d7509cda80015b3e43e1a256cddd671 Mon Sep 17 00:00:00 2001
From: root <root@p006vm88.pbm.ihost.com>
Date: Fri, 11 Feb 2022 10:24:56 +0000
Subject: [PATCH 052/210] resolved format error

---
 flow/ThreadPrimitives.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flow/ThreadPrimitives.h b/flow/ThreadPrimitives.h
index 44faa72eff..c0913a23c9 100644
--- a/flow/ThreadPrimitives.h
+++ b/flow/ThreadPrimitives.h
@@ -65,11 +65,11 @@ public:
 	void enter() {
 		while (isLocked.test_and_set(std::memory_order_acquire))
 #if defined(__aarch64__)
-		__asm__ volatile("isb");
+			__asm__ volatile("isb");
 #elif defined(__powerpc64__)
-		__asm__ volatile ("or 27,27,27" ::: "memory");
+			__asm__ volatile ("or 27,27,27" ::: "memory");
 #else
-		_mm_pause();
+			_mm_pause();
 #endif
 #if VALGRIND
 		ANNOTATE_RWLOCK_ACQUIRED(this, true);

From 459dd8358360b0f0e0204f1fbe32d73e1e6fc22c Mon Sep 17 00:00:00 2001
From: root <root@p006vm88.pbm.ihost.com>
Date: Fri, 11 Feb 2022 12:11:50 +0000
Subject: [PATCH 053/210] resolved format error

---
 flow/Platform.h         | 4 +---
 flow/ThreadPrimitives.h | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/flow/Platform.h b/flow/Platform.h
index 24e10d59f8..b5e5a64878 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -472,9 +472,7 @@ inline static uint64_t __rdtsc() {
 #elif defined(__powerpc64__) || defined(__ppc64__)
 inline static uint64_t __rdtsc() {
 	uint64_t tb;
-	__asm__ volatile (\
-			"mfspr %0, 268"
-			: "=r" (tb));
+	__asm__ volatile("mfspr %0, 268" : "=r"(tb));
 	return tb;
 }
 #endif
diff --git a/flow/ThreadPrimitives.h b/flow/ThreadPrimitives.h
index c0913a23c9..7ee3cc90f1 100644
--- a/flow/ThreadPrimitives.h
+++ b/flow/ThreadPrimitives.h
@@ -67,7 +67,7 @@ public:
 #if defined(__aarch64__)
 			__asm__ volatile("isb");
 #elif defined(__powerpc64__)
-			__asm__ volatile ("or 27,27,27" ::: "memory");
+			__asm__ volatile("or 27,27,27" ::: "memory");
 #else
 			_mm_pause();
 #endif

From 9bf11ac2f354feef6469c25a78362c400cd9e394 Mon Sep 17 00:00:00 2001
From: root <root@p006vm88.pbm.ihost.com>
Date: Fri, 11 Feb 2022 13:45:17 +0000
Subject: [PATCH 054/210] Removed condition for arch64

---
 flow/Platform.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Platform.h b/flow/Platform.h
index b5e5a64878..505e475c58 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -842,7 +842,7 @@ static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
 #endif
 #endif
 
-#if !defined(__aarch64__) && !defined(__powerpc64__)
+#if defined(__aarch64__) || defined(__powerpc64__)
 #define _MM_HINT_T0 0 /* dummy -- not used */
 #endif
 

From a63d218e9de38530da56bead5e5b7500b610912f Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Fri, 11 Feb 2022 16:41:14 -0500
Subject: [PATCH 055/210] simplify test workload and adjust ratekeeper
 throttling strategy

---
 fdbclient/CommitProxyInterface.h             | 17 +++---
 fdbclient/DatabaseContext.h                  |  3 +-
 fdbclient/NativeAPI.actor.cpp                | 36 +++++++++----
 fdbserver/GrvProxyServer.actor.cpp           | 55 ++++++++++----------
 fdbserver/workloads/SidebandSingle.actor.cpp | 51 +-----------------
 5 files changed, 64 insertions(+), 98 deletions(-)

diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index 7e670ed08a..5556579992 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -198,8 +198,8 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 	bool locked;
 	Optional<Value> metadataVersion;
 	int64_t midShardSize = 0;
-	uint32_t queueIterations = 0;
-	bool rkThrottled = false;
+	bool rkDefaultThrottled = false;
+	bool rkBatchThrottled = false;
 
 	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
 
@@ -214,8 +214,8 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
 		           metadataVersion,
 		           tagThrottleInfo,
 		           midShardSize,
-		           queueIterations,
-		           rkThrottled);
+		           rkDefaultThrottled,
+		           rkBatchThrottled);
 	}
 };
 
@@ -238,22 +238,21 @@ struct GetReadVersionRequest : TimedRequest {
 	uint32_t transactionCount;
 	uint32_t flags;
 	TransactionPriority priority;
-	uint32_t queueIterations;
 
 	TransactionTagMap<uint32_t> tags;
 
 	Optional<UID> debugID;
 	ReplyPromise<GetReadVersionReply> reply;
 
-	GetReadVersionRequest() : transactionCount(1), flags(0), queueIterations(0) {}
+	GetReadVersionRequest() : transactionCount(1), flags(0) {}
 	GetReadVersionRequest(SpanID spanContext,
 	                      uint32_t transactionCount,
 	                      TransactionPriority priority,
 	                      uint32_t flags = 0,
 	                      TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
 	                      Optional<UID> debugID = Optional<UID>())
-	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority),
-	    queueIterations(0), tags(tags), debugID(debugID) {
+	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags),
+	    debugID(debugID) {
 		flags = flags & ~FLAG_PRIORITY_MASK;
 		switch (priority) {
 		case TransactionPriority::BATCH:
@@ -274,7 +273,7 @@ struct GetReadVersionRequest : TimedRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, queueIterations);
+		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext);
 
 		if (ar.isDeserializing) {
 			if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 0ba157a04d..959e7decfe 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -475,7 +475,8 @@ public:
 	void updateCachedReadVersion(double t, Version v);
 	Version getCachedReadVersion();
 	double getLastGrvTime();
-	double lastRkThrottleTime;
+	double lastRkBatchThrottleTime;
+	double lastRkDefaultThrottleTime;
 	// Cached RVs can be updated through commits, and using cached RVs avoids the proxies altogether
 	// Because our checks for ratekeeper throttling requires communication with the proxies,
 	// we want to track the last time in order to periodically contact the proxy to check for throttling
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4d05bf2974..8070c23a66 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1306,10 +1306,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
     latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), outstandingWatches(0), lastGrvTime(0.0), cachedReadVersion(0), lastRkThrottleTime(0.0),
-    lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
-    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    bytesPerCommit(1000), outstandingWatches(0), lastGrvTime(0.0), cachedReadVersion(0), lastRkBatchThrottleTime(0.0),
+    lastRkDefaultThrottleTime(0.0), lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID),
+    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
+    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)),
     connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {
@@ -5963,11 +5963,15 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
                                          Promise<Optional<Value>> metadataVersion) {
 	state Span span(spanContext, location, { trState->spanID });
 	GetReadVersionReply rep = wait(f);
-	double latency = now() - trState->startTime;
+	double replyTime = now();
+	double latency = replyTime - trState->startTime;
 	trState->cx->lastProxyRequestTime = trState->startTime;
 	trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
-	if (rep.rkThrottled && trState->options.priority != TransactionPriority::IMMEDIATE) {
-		trState->cx->lastRkThrottleTime = now();
+	if (rep.rkBatchThrottled) {
+		trState->cx->lastRkBatchThrottleTime = replyTime;
+	}
+	if (rep.rkDefaultThrottled) {
+		trState->cx->lastRkDefaultThrottleTime = replyTime;
 	}
 	trState->cx->GRVLatencies.addSample(latency);
 	if (trState->trLogInfo)
@@ -6025,11 +6029,21 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	return rep.version;
 }
 
-bool rkThrottlingCooledDown(DatabaseContext* cx) {
-	if (cx->lastRkThrottleTime == 0.0) {
+bool rkThrottlingCooledDown(DatabaseContext* cx, TransactionPriority priority) {
+	if (priority == TransactionPriority::IMMEDIATE) {
 		return true;
+	} else if (priority == TransactionPriority::BATCH) {
+		if (cx->lastRkBatchThrottleTime == 0.0) {
+			return true;
+		}
+		return (now() - cx->lastRkBatchThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
+	} else if (priority == TransactionPriority::DEFAULT) {
+		if (cx->lastRkDefaultThrottleTime == 0.0) {
+			return true;
+		}
+		return (now() - cx->lastRkDefaultThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
 	}
-	return (now() - cx->lastRkThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
+	return false;
 }
 
 Future<Version> Transaction::getReadVersion(uint32_t flags) {
@@ -6037,7 +6051,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !trState->options.skipGrvCache &&
 		    (deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE ||
 		     trState->options.useGrvCache) &&
-		    rkThrottlingCooledDown(getDatabase().getPtr())) {
+		    rkThrottlingCooledDown(getDatabase().getPtr(), trState->options.priority)) {
 			// Upon our first request to use cached RVs, start the background updater
 			if (!trState->cx->grvUpdateHandler.isValid()) {
 				trState->cx->grvUpdateHandler = backgroundGrvUpdater(getDatabase().getPtr());
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index a3fe28b002..82692728bd 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -47,8 +47,10 @@ struct GrvProxyStats {
 	double percentageOfDefaultGRVQueueProcessed;
 	double percentageOfBatchGRVQueueProcessed;
 
-	bool lastTxnThrottled;
-	double throttleStartTime;
+	bool lastBatchQueueThrottled;
+	bool lastDefaultQueueThrottled;
+	double batchThrottleStartTime;
+	double defaultThrottleStartTime;
 
 	LatencySample defaultTxnGRVTimeInQueue;
 	LatencySample batchTxnGRVTimeInQueue;
@@ -102,7 +104,8 @@ struct GrvProxyStats {
 	    updatesFromRatekeeper("UpdatesFromRatekeeper", cc), leaseTimeouts("LeaseTimeouts", cc), systemGRVQueueSize(0),
 	    defaultGRVQueueSize(0), batchGRVQueueSize(0), transactionRateAllowed(0), batchTransactionRateAllowed(0),
 	    transactionLimit(0), batchTransactionLimit(0), percentageOfDefaultGRVQueueProcessed(0),
-	    percentageOfBatchGRVQueueProcessed(0), lastTxnThrottled(false), throttleStartTime(0.0),
+	    percentageOfBatchGRVQueueProcessed(0), lastBatchQueueThrottled(false), lastDefaultQueueThrottled(false),
+	    batchThrottleStartTime(0.0), defaultThrottleStartTime(0.0),
 	    defaultTxnGRVTimeInQueue("DefaultTxnGRVTimeInQueue",
 	                             id,
 	                             SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
@@ -653,31 +656,16 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			}
 		}
 
-		reply.queueIterations = request.queueIterations;
-		TraceEvent(SevDebug, "DebugGrvProxyThrottleCheck")
-		    .detail("QueueIterations", reply.queueIterations)
-		    .detail("ThrottleThreshold", CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD)
-		    .detail("LastTxnThrottled", stats->lastTxnThrottled)
-		    .detail("ThrottleStartTime", format("%.6f", stats->throttleStartTime))
-		    .detail("Diff", now() - stats->throttleStartTime)
-		    .detail("SustainedThrottlingThreshold", CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD);
-		if (reply.queueIterations >= CLIENT_KNOBS->GRV_THROTTLING_THRESHOLD) {
-			if (stats->lastTxnThrottled) {
-				// Check if this throttling has been sustained for a certain amount of time to avoid false positives
-				if (now() - stats->throttleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
-					reply.rkThrottled = true;
-				}
-			} else { // !stats->lastTxnThrottled
-				// If not previously throttled, this request/reply is our new starting point
-				// for judging whether we are being actively throttled by ratekeeper now
-				stats->lastTxnThrottled = true;
-				stats->throttleStartTime = now();
+		if (stats->lastBatchQueueThrottled) {
+			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
+			if (now() - stats->batchThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+				reply.rkBatchThrottled = true;
 			}
-		} else {
-			// If an immediate priority txn comes in, it may get processed on the first time through and
-			// reset our throttling state prematurely.
-			if (request.priority != TransactionPriority::IMMEDIATE) {
-				stats->lastTxnThrottled = false;
+		}
+		if (stats->lastDefaultQueueThrottled) {
+			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
+			if (now() - stats->defaultThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+				reply.rkDefaultThrottled = true;
 			}
 		}
 		request.reply.send(reply);
@@ -841,7 +829,6 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			// transactionQueue->span.swap(span);
 
 			auto& req = transactionQueue->front();
-			req.queueIterations++;
 			int tc = req.transactionCount;
 
 			if (req.priority < TransactionPriority::DEFAULT &&
@@ -877,6 +864,18 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			transactionQueue->pop_front();
 			requestsToStart++;
 		}
+		if (!batchQueue.empty()) {
+			grvProxyData->stats.lastBatchQueueThrottled = true;
+			grvProxyData->stats.batchThrottleStartTime = now();
+		} else {
+			grvProxyData->stats.lastBatchQueueThrottled = false;
+		}
+		if (!defaultQueue.empty()) {
+			grvProxyData->stats.lastDefaultQueueThrottled = true;
+			grvProxyData->stats.defaultThrottleStartTime = now();
+		} else {
+			grvProxyData->stats.lastDefaultQueueThrottled = false;
+		}
 
 		if (!systemQueue.empty() || !defaultQueue.empty() || !batchQueue.empty()) {
 			forwardPromise(
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index 0ec80a9ef9..7a82b20c12 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -74,11 +74,7 @@ struct SidebandSingleWorkload : TestWorkload {
 	}
 
 	std::string description() const override { return "SidebandSingleWorkload"; }
-	Future<Void> setup(Database const& cx) override {
-		if (clientId != 0)
-			return Void();
-		return persistInterface(this, cx);
-	}
+	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
@@ -108,51 +104,8 @@ struct SidebandSingleWorkload : TestWorkload {
 		m.push_back(keysUnexpectedlyPresent.getMetric());
 	}
 
-	ACTOR Future<Void> persistInterface(SidebandSingleWorkload* self, Database cx) {
-		state Transaction tr(cx);
-		BinaryWriter wr(IncludeVersion());
-		wr << self->interf;
-		state Standalone<StringRef> serializedInterface = wr.toValue();
-		loop {
-			try {
-				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
-				if (val.present()) {
-					if (val.get() != serializedInterface)
-						throw operation_failed();
-					break;
-				}
-				tr.set(format("Sideband/Client/%d", self->clientId), serializedInterface);
-				wait(tr.commit());
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-		TraceEvent("SidebandPersisted", self->interf.id()).detail("ClientIdx", self->clientId);
-		return Void();
-	}
-
-	ACTOR Future<SidebandInterface> fetchSideband(SidebandSingleWorkload* self, Database cx) {
-		state Transaction tr(cx);
-		loop {
-			try {
-				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
-				if (!val.present()) {
-					throw operation_failed();
-				}
-				SidebandInterface sideband;
-				BinaryReader br(val.get(), IncludeVersion());
-				br >> sideband;
-				TraceEvent("SidebandFetched", sideband.id()).detail("ClientIdx", self->clientId);
-				return sideband;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
 	ACTOR Future<Void> mutator(SidebandSingleWorkload* self, Database cx) {
-		state SidebandInterface checker = wait(self->fetchSideband(self, cx));
+		state SidebandInterface checker = self->interf;
 		state double lastTime = now();
 		state Version commitVersion;
 		state bool unknown = false;

From 22cf6d8e0e6fe89d050d21831be8a05cb409b9ae Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Sun, 13 Feb 2022 16:10:45 -0800
Subject: [PATCH 056/210] Revert "Revert "add proper downgrade/upgrade test for
 storage migration with PPW""

This reverts commit 6d3b7c97b842fe03ea2f74ce021fea5b96323826.
---
 tests/CMakeLists.txt                          |  8 ++++++
 ...onfigureStorageMigrationTestRestart-1.toml | 25 +++++++++++++++++++
 ...onfigureStorageMigrationTestRestart-2.toml | 19 ++++++++++++++
 ...onfigureStorageMigrationTestRestart-1.toml | 25 +++++++++++++++++++
 ...onfigureStorageMigrationTestRestart-2.toml | 19 ++++++++++++++
 5 files changed, 96 insertions(+)
 create mode 100644 tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
 create mode 100644 tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
 create mode 100644 tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
 create mode 100644 tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index df6cf64466..6c52b440fa 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -243,6 +243,9 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/to_7.0.0/CycleTestRestart-1.txt
                restarting/to_7.0.0/CycleTestRestart-2.txt)
+  add_fdb_test(
+    TEST_FILES restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
+               restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml)
   add_fdb_test(
     TEST_FILES restarting/from_7.0.0/SnapTestAttrition-1.txt
                restarting/from_7.0.0/SnapTestAttrition-2.txt)
@@ -255,6 +258,11 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/from_7.0.0/SnapCycleRestart-1.txt
                restarting/from_7.0.0/SnapCycleRestart-2.txt)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
+               restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml)
+
+
   add_fdb_test(TEST_FILES slow/ApiCorrectness.toml)
   add_fdb_test(TEST_FILES slow/ApiCorrectnessAtomicRestore.toml)
   add_fdb_test(TEST_FILES slow/ApiCorrectnessSwitchover.toml)
diff --git a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
new file mode 100644
index 0000000000..a70a9e70bf
--- /dev/null
+++ b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
@@ -0,0 +1,25 @@
+extraMachineCountDC = 2
+
+[[test]]
+testTitle = 'CloggedConfigureDatabaseTest'
+clearAfterTest = false
+
+    [[test.workload]]
+    testName = 'ConfigureDatabase'
+    testDuration = 30.0
+    allowTestStorageMigration = true
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+    scale = 0.1
+    clogginess = 2.0
+
+    [[test.workload]]
+    testName='SaveAndKill'
+    restartInfoLocation='simfdb/restartInfo.ini'
+    testDuration=30.0
\ No newline at end of file
diff --git a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
new file mode 100644
index 0000000000..f8361ab8c7
--- /dev/null
+++ b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
@@ -0,0 +1,19 @@
+extraMachineCountDC = 2
+
+[[test]]
+testTitle = 'CloggedConfigureDatabaseTest'
+runSetup=false
+
+    [[test.workload]]
+    testName = 'ConfigureDatabase'
+    testDuration = 300.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 300.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 300.0
+    scale = 0.1
+    clogginess = 2.0
\ No newline at end of file
diff --git a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
new file mode 100644
index 0000000000..a70a9e70bf
--- /dev/null
+++ b/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
@@ -0,0 +1,25 @@
+extraMachineCountDC = 2
+
+[[test]]
+testTitle = 'CloggedConfigureDatabaseTest'
+clearAfterTest = false
+
+    [[test.workload]]
+    testName = 'ConfigureDatabase'
+    testDuration = 30.0
+    allowTestStorageMigration = true
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+    scale = 0.1
+    clogginess = 2.0
+
+    [[test.workload]]
+    testName='SaveAndKill'
+    restartInfoLocation='simfdb/restartInfo.ini'
+    testDuration=30.0
\ No newline at end of file
diff --git a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
new file mode 100644
index 0000000000..f8361ab8c7
--- /dev/null
+++ b/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
@@ -0,0 +1,19 @@
+extraMachineCountDC = 2
+
+[[test]]
+testTitle = 'CloggedConfigureDatabaseTest'
+runSetup=false
+
+    [[test.workload]]
+    testName = 'ConfigureDatabase'
+    testDuration = 300.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 300.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 300.0
+    scale = 0.1
+    clogginess = 2.0
\ No newline at end of file

From 5445dd3106eadd00d719f3a93e714b1f7d046564 Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Sun, 13 Feb 2022 23:30:14 -0800
Subject: [PATCH 057/210] add option to fix consistency check

---
 fdbserver/workloads/ConfigureDatabase.actor.cpp               | 4 +++-
 .../from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml    | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp
index fde8e8f76e..c9ea1f1895 100644
--- a/fdbserver/workloads/ConfigureDatabase.actor.cpp
+++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp
@@ -220,6 +220,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	int additionalDBs;
 	bool allowDescriptorChange;
 	bool allowTestStorageMigration;
+	bool waitStoreTypeCheck;
 	std::vector<Future<Void>> clients;
 	PerfIntCounter retries;
 
@@ -229,6 +230,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 		    getOption(options, LiteralStringRef("allowDescriptorChange"), SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
 		allowTestStorageMigration =
 		    getOption(options, "allowTestStorageMigration"_sr, false) && g_simulator.allowStorageMigrationTypeChange;
+		waitStoreTypeCheck = getOption(options, "waitStoreTypeCheck"_sr, false);
 		g_simulator.usableRegions = 1;
 	}
 
@@ -273,7 +275,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 		// only storage_migration_type=gradual && perpetual_storage_wiggle=1 need this check because in QuietDatabase
 		// perpetual wiggle will be forced to close For other cases, later ConsistencyCheck will check KV store type
 		// there
-		if (self->allowTestStorageMigration) {
+		if (self->allowTestStorageMigration || self->waitStoreTypeCheck) {
 			loop {
 				// There exists a race where the check can start before the last transaction that singleDB issued
 				// finishes, if singleDB gets actor cancelled from a timeout at the end of a test. This means the
diff --git a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
index f8361ab8c7..aac15a614d 100644
--- a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
+++ b/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
@@ -3,10 +3,12 @@ extraMachineCountDC = 2
 [[test]]
 testTitle = 'CloggedConfigureDatabaseTest'
 runSetup=false
+waitForQuiescenceBegin=false
 
     [[test.workload]]
     testName = 'ConfigureDatabase'
     testDuration = 300.0
+    waitStoreTypeCheck = true
 
     [[test.workload]]
     testName = 'RandomClogging'

From 8129c4e21cbb0db3ecfe85c8d00273ea880d8a83 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Mon, 14 Feb 2022 13:58:56 -0500
Subject: [PATCH 058/210] simplify sidebandsingle workload and be stricter with
 batch throttling on rk

---
 fdbserver/GrvProxyServer.actor.cpp           |  4 ++
 fdbserver/workloads/SidebandSingle.actor.cpp | 51 +++++---------------
 2 files changed, 17 insertions(+), 38 deletions(-)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 82692728bd..6fe559d02d 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -665,6 +665,10 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		if (stats->lastDefaultQueueThrottled) {
 			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 			if (now() - stats->defaultThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+				// Consider the batch queue throttled if the default is throttled
+				// to deal with a potential lull in activity for that priority.
+				// Avoids mistakenly thinking batch is unthrottled while default is still throttled.
+				reply.rkBatchThrottled = true;
 				reply.rkDefaultThrottled = true;
 			}
 		}
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index 7a82b20c12..e92adb75f2 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -33,35 +33,10 @@
  * unknown result matches the regular read of that same key and is not too stale.
  */
 
-struct SidebandMessage {
-	constexpr static FileIdentifier file_identifier = 11862047;
-	uint64_t key;
-	Version commitVersion;
-
-	SidebandMessage() {}
-	SidebandMessage(uint64_t key, Version commitVersion) : key(key), commitVersion(commitVersion) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, key, commitVersion);
-	}
-};
-
-struct SidebandInterface {
-	constexpr static FileIdentifier file_identifier = 15950545;
-	RequestStream<SidebandMessage> updates;
-
-	UID id() const { return updates.getEndpoint().token; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, updates);
-	}
-};
-
 struct SidebandSingleWorkload : TestWorkload {
 	double testDuration, operationsPerSecond;
-	SidebandInterface interf;
+	// Pair represents <Key, commitVersion>
+	PromiseStream<std::pair<uint64_t, Version>> interf;
 
 	std::vector<Future<Void>> clients;
 	PerfIntCounter messages, consistencyErrors, keysUnexpectedlyPresent;
@@ -105,7 +80,6 @@ struct SidebandSingleWorkload : TestWorkload {
 	}
 
 	ACTOR Future<Void> mutator(SidebandSingleWorkload* self, Database cx) {
-		state SidebandInterface checker = self->interf;
 		state double lastTime = now();
 		state Version commitVersion;
 		state bool unknown = false;
@@ -138,7 +112,7 @@ struct SidebandSingleWorkload : TestWorkload {
 					if (e.code() == error_code_commit_unknown_result) {
 						unknown = true;
 						++self->messages;
-						checker.updates.send(SidebandMessage(key, invalidVersion));
+						self->interf.send(std::pair(key, invalidVersion));
 						break;
 					}
 					wait(tr.onError(e));
@@ -149,31 +123,32 @@ struct SidebandSingleWorkload : TestWorkload {
 				continue;
 			}
 			++self->messages;
-			checker.updates.send(SidebandMessage(key, commitVersion));
+			self->interf.send(std::pair(key, commitVersion));
 		}
 	}
 
 	ACTOR Future<Void> checker(SidebandSingleWorkload* self, Database cx) {
 		loop {
-			state SidebandMessage message = waitNext(self->interf.updates.getFuture());
-			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.key));
+			// Pair represents <Key, commitVersion>
+			state std::pair<uint64_t, Version> message = waitNext(self->interf.getFuture());
+			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.first));
 			state Transaction tr(cx);
 			loop {
 				try {
 					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
 					state Optional<Value> val = wait(tr.get(messageKey));
 					if (!val.present()) {
-						TraceEvent(SevError, "CausalConsistencyError1", self->interf.id())
+						TraceEvent(SevError, "CausalConsistencyError1")
 						    .detail("MessageKey", messageKey.toString().c_str())
-						    .detail("RemoteCommitVersion", message.commitVersion)
+						    .detail("RemoteCommitVersion", message.second)
 						    .detail("LocalReadVersion",
 						            tr.getReadVersion().get()); // will assert that ReadVersion is set
 						++self->consistencyErrors;
 					} else if (val.get() != LiteralStringRef("deadbeef")) {
 						// If we read something NOT "deadbeef" and there was no commit_unknown_result,
 						// the cache somehow read a stale version of our key
-						if (message.commitVersion != invalidVersion) {
-							TraceEvent(SevError, "CausalConsistencyError2", self->interf.id())
+						if (message.second != invalidVersion) {
+							TraceEvent(SevError, "CausalConsistencyError2")
 							    .detail("MessageKey", messageKey.toString().c_str());
 							++self->consistencyErrors;
 							break;
@@ -191,11 +166,11 @@ struct SidebandSingleWorkload : TestWorkload {
 							}
 						}
 						if (val != val2) {
-							TraceEvent(SevError, "CausalConsistencyError3", self->interf.id())
+							TraceEvent(SevError, "CausalConsistencyError3")
 							    .detail("MessageKey", messageKey.toString().c_str())
 							    .detail("Val1", val)
 							    .detail("Val2", val2)
-							    .detail("RemoteCommitVersion", message.commitVersion)
+							    .detail("RemoteCommitVersion", message.second)
 							    .detail("LocalReadVersion",
 							            tr.getReadVersion().get()); // will assert that ReadVersion is set
 							++self->consistencyErrors;

From 9beae3fb64123d472078b0c58c466c7dd9c4b699 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 11:49:45 -0800
Subject: [PATCH 059/210] Add RatekeeperData class with refactored
 implementation

---
 fdbserver/CMakeLists.txt           |   2 +
 fdbserver/RatekeeperData.actor.cpp | 865 +++++++++++++++++++++++++++++
 fdbserver/RatekeeperData.h         | 568 +++++++++++++++++++
 3 files changed, 1435 insertions(+)
 create mode 100644 fdbserver/RatekeeperData.actor.cpp
 create mode 100644 fdbserver/RatekeeperData.h

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index c207ee694f..38d3f18249 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -91,6 +91,8 @@ set(FDBSERVER_SRCS
   QuietDatabase.h
   RadixTree.h
   Ratekeeper.actor.cpp
+  RatekeeperData.actor.cpp
+  RatekeeperData.h
   RatekeeperInterface.h
   RecoveryState.h
   RestoreCommon.actor.h
diff --git a/fdbserver/RatekeeperData.actor.cpp b/fdbserver/RatekeeperData.actor.cpp
new file mode 100644
index 0000000000..09e4c56f1f
--- /dev/null
+++ b/fdbserver/RatekeeperData.actor.cpp
@@ -0,0 +1,865 @@
+/*
+ * RatekeeperData.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/RatekeeperData.h"
+#include "flow/actorcompiler.h" // must be last include
+
+ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
+	try {
+		wait(in);
+		return Void();
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && !errOut.isSet())
+			errOut.sendError(e);
+		throw;
+	}
+}
+
+class RatekeeperDataImpl {
+public:
+	ACTOR static Future<Void> configurationMonitor(RatekeeperData* self) {
+		loop {
+			state ReadYourWritesTransaction tr(self->db);
+
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+					ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+
+					self->configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
+
+					state Future<Void> watchFuture =
+					    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
+					    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
+					    tr.watch(failedLocalityVersionKey);
+					wait(tr.commit());
+					wait(watchFuture);
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> monitorServerListChange(
+	    RatekeeperData* self,
+	    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+		state std::map<UID, StorageServerInterface> oldServers;
+		state Transaction tr(self->db);
+
+		loop {
+			try {
+				if (now() - self->lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
+					TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", self->id)
+					    .detail("Latency", now() - self->lastSSListFetchedTimestamp);
+				}
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
+				    wait(getServerListAndProcessClasses(&tr));
+				self->lastSSListFetchedTimestamp = now();
+
+				std::map<UID, StorageServerInterface> newServers;
+				for (const auto& [ssi, _] : results) {
+					const UID serverId = ssi.id();
+					newServers[serverId] = ssi;
+
+					if (oldServers.count(serverId)) {
+						if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) {
+							serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
+						}
+						oldServers.erase(serverId);
+					} else {
+						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
+					}
+				}
+
+				for (const auto& it : oldServers) {
+					serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
+				}
+
+				oldServers.swap(newServers);
+				tr = Transaction(self->db);
+				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
+			} catch (Error& e) {
+				TraceEvent("RatekeeperGetSSListError", self->id).error(e).suppressFor(1.0);
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> trackStorageServerQueueInfo(RatekeeperData* self, StorageServerInterface ssi) {
+		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
+		state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
+		TraceEvent("RkTracking", self->id)
+		    .detail("StorageServer", ssi.id())
+		    .detail("Locality", ssi.locality.toString());
+		try {
+			loop {
+				ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
+				    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
+				if (reply.present()) {
+					myQueueInfo->value.valid = true;
+					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
+					myQueueInfo->value.lastReply = reply.get();
+					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
+						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
+						myQueueInfo->value.smoothDurableVersion.reset(reply.get().durableVersion);
+						myQueueInfo->value.smoothLatestVersion.reset(reply.get().version);
+					} else {
+						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
+						                                       myQueueInfo->value.prevReply.bytesDurable);
+						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
+						myQueueInfo->value.smoothDurableVersion.setTotal(reply.get().durableVersion);
+						myQueueInfo->value.smoothLatestVersion.setTotal(reply.get().version);
+					}
+
+					myQueueInfo->value.busiestReadTag = reply.get().busiestTag;
+					myQueueInfo->value.busiestReadTagFractionalBusyness = reply.get().busiestTagFractionalBusyness;
+					myQueueInfo->value.busiestReadTagRate = reply.get().busiestTagRate;
+				} else {
+					if (myQueueInfo->value.valid) {
+						TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id());
+					}
+					myQueueInfo->value.valid = false;
+				}
+
+				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+				     IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
+				                                                    FailureStatus(false)));
+			}
+		} catch (...) {
+			// including cancellation
+			self->storageQueueInfo.erase(myQueueInfo);
+			throw;
+		}
+	}
+
+	ACTOR static Future<Void> trackTLogQueueInfo(RatekeeperData* self, TLogInterface tli) {
+		self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
+		state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
+		TraceEvent("RkTracking", self->id).detail("TransactionLog", tli.id());
+		try {
+			loop {
+				ErrorOr<TLogQueuingMetricsReply> reply = wait(tli.getQueuingMetrics.getReplyUnlessFailedFor(
+				    TLogQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
+				if (reply.present()) {
+					myQueueInfo->value.valid = true;
+					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
+					myQueueInfo->value.lastReply = reply.get();
+					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
+						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
+					} else {
+						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
+						                                       myQueueInfo->value.prevReply.bytesDurable);
+						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
+					}
+				} else {
+					if (myQueueInfo->value.valid) {
+						TraceEvent("RkTLogDidNotRespond", self->id).detail("TransactionLog", tli.id());
+					}
+					myQueueInfo->value.valid = false;
+				}
+
+				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+				     IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
+				                                                    FailureStatus(false)));
+			}
+		} catch (...) {
+			// including cancellation
+			self->tlogQueueInfo.erase(myQueueInfo);
+			throw;
+		}
+	}
+
+	ACTOR static Future<Void> trackEachStorageServer(
+	    RatekeeperData* self,
+	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+		state Map<UID, Future<Void>> actors;
+		state Promise<Void> err;
+		loop choose {
+			when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
+				wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
+				if (change.second.present()) {
+					if (!change.second.get().isTss()) {
+						auto& a = actors[change.first];
+						a = Future<Void>();
+						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
+					}
+				} else
+					actors.erase(change.first);
+			}
+			when(wait(err.getFuture())) {}
+		}
+	}
+
+}; // class RatekeeperDataImpl
+
+Future<Void> RatekeeperData::configurationMonitor() {
+	return RatekeeperDataImpl::configurationMonitor(this);
+}
+
+Future<Void> RatekeeperData::monitorServerListChange(
+    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	return RatekeeperDataImpl::monitorServerListChange(this, serverChanges);
+}
+
+Future<Void> RatekeeperData::trackEachStorageServer(
+    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	return RatekeeperDataImpl::trackEachStorageServer(this, serverChanges);
+}
+
+Future<Void> RatekeeperData::trackStorageServerQueueInfo(StorageServerInterface ssi) {
+	return RatekeeperDataImpl::trackStorageServerQueueInfo(this, ssi);
+}
+
+Future<Void> RatekeeperData::trackTLogQueueInfo(TLogInterface tli) {
+	return RatekeeperDataImpl::trackTLogQueueInfo(this, tli);
+}
+
+RatekeeperData::RatekeeperData(UID id, Database db)
+  : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
+    actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
+    lastBusiestCommitTagPick(0), throttledTagChangeId(0), normalLimits(TransactionPriority::DEFAULT,
+                                                                       "",
+                                                                       SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
+                                                                       SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
+                                                                       SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
+                                                                       SERVER_KNOBS->SPRING_BYTES_TLOG,
+                                                                       SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
+                                                                       SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
+    batchLimits(TransactionPriority::BATCH,
+                "Batch",
+                SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
+                SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH,
+                SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
+                SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
+                SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
+    autoThrottlingEnabled(false) {
+	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
+	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
+}
+
+void RatekeeperData::updateCommitCostEstimation(
+    UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation) {
+	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
+		auto tagCostIt = costEstimation.find(it->key);
+		if (tagCostIt == costEstimation.end())
+			continue;
+		for (const auto& [tagName, cost] : tagCostIt->second) {
+			it->value.tagCostEst[tagName] += cost;
+			it->value.totalWriteCosts += cost.getCostSum();
+			it->value.totalWriteOps += cost.getOpsSum();
+		}
+	}
+}
+
+void RatekeeperData::updateRate(RatekeeperLimits* limits) {
+	// double controlFactor = ;  // dt / eFoldingTime
+
+	double actualTps = smoothReleasedTransactions.smoothRate();
+	actualTpsMetric = (int64_t)actualTps;
+	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this
+	// value
+	actualTps =
+	    std::max(std::max(1.0, actualTps), smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
+
+	if (actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
+		actualTpsHistory.pop_front();
+	}
+	actualTpsHistory.push_back(actualTps);
+
+	limits->tpsLimit = std::numeric_limits<double>::infinity();
+	UID reasonID = UID();
+	limitReason_t limitReason = limitReason_t::unlimited;
+
+	int sscount = 0;
+
+	int64_t worstFreeSpaceStorageServer = std::numeric_limits<int64_t>::max();
+	int64_t worstStorageQueueStorageServer = 0;
+	int64_t limitingStorageQueueStorageServer = 0;
+	int64_t worstDurabilityLag = 0;
+
+	std::multimap<double, StorageQueueInfo*> storageTpsLimitReverseIndex;
+	std::multimap<int64_t, StorageQueueInfo*> storageDurabilityLagReverseIndex;
+
+	std::map<UID, limitReason_t> ssReasons;
+
+	bool printRateKeepLimitReasonDetails =
+	    SERVER_KNOBS->RATEKEEPER_PRINT_LIMIT_REASON &&
+	    (deterministicRandom()->random01() < SERVER_KNOBS->RATEKEEPER_LIMIT_REASON_SAMPLE_RATE);
+
+	// Look at each storage server's write queue and local rate, compute and store the desired rate ratio
+	for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) {
+		auto& ss = i->value;
+		if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
+			continue;
+		++sscount;
+
+		limitReason_t ssLimitReason = limitReason_t::unlimited;
+
+		int64_t minFreeSpace =
+		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
+		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * ss.smoothTotalSpace.smoothTotal()));
+
+		worstFreeSpaceStorageServer =
+		    std::min(worstFreeSpaceStorageServer, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace);
+
+		int64_t springBytes = std::max<int64_t>(
+		    1, std::min<int64_t>(limits->storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
+		int64_t targetBytes = std::max<int64_t>(
+		    1, std::min(limits->storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
+		if (targetBytes != limits->storageTargetBytes) {
+			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
+				ssLimitReason = limitReason_t::storage_server_min_free_space;
+			} else {
+				ssLimitReason = limitReason_t::storage_server_min_free_space_ratio;
+			}
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("Reason", ssLimitReason)
+				    .detail("SSID", ss.id)
+				    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
+				    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
+				    .detail("TargetBytes", targetBytes)
+				    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
+				    .detail("MinFreeSpace", minFreeSpace);
+			}
+		}
+
+		int64_t storageQueue = ss.lastReply.bytesInput - ss.smoothDurableBytes.smoothTotal();
+		worstStorageQueueStorageServer = std::max(worstStorageQueueStorageServer, storageQueue);
+
+		int64_t storageDurabilityLag = ss.smoothLatestVersion.smoothTotal() - ss.smoothDurableVersion.smoothTotal();
+		worstDurabilityLag = std::max(worstDurabilityLag, storageDurabilityLag);
+
+		storageDurabilityLagReverseIndex.insert(std::make_pair(-1 * storageDurabilityLag, &ss));
+
+		auto& ssMetrics = healthMetrics.storageStats[ss.id];
+		ssMetrics.storageQueue = storageQueue;
+		ssMetrics.storageDurabilityLag = storageDurabilityLag;
+		ssMetrics.cpuUsage = ss.lastReply.cpuUsage;
+		ssMetrics.diskUsage = ss.lastReply.diskUsage;
+
+		double targetRateRatio = std::min((storageQueue - targetBytes + springBytes) / (double)springBytes, 2.0);
+
+		if (limits->priority == TransactionPriority::DEFAULT) {
+			tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag);
+		}
+
+		double inputRate = ss.smoothInputBytes.smoothRate();
+		// inputRate = std::max( inputRate, actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE );
+
+		/*if( deterministicRandom()->random01() < 0.1 ) {
+		  std::string name = "RatekeeperUpdateRate" + limits.context;
+		  TraceEvent(name, ss.id)
+		  .detail("MinFreeSpace", minFreeSpace)
+		  .detail("SpringBytes", springBytes)
+		  .detail("TargetBytes", targetBytes)
+		  .detail("SmoothTotalSpaceTotal", ss.smoothTotalSpace.smoothTotal())
+		  .detail("SmoothFreeSpaceTotal", ss.smoothFreeSpace.smoothTotal())
+		  .detail("LastReplyBytesInput", ss.lastReply.bytesInput)
+		  .detail("SmoothDurableBytesTotal", ss.smoothDurableBytes.smoothTotal())
+		  .detail("TargetRateRatio", targetRateRatio)
+		  .detail("SmoothInputBytesRate", ss.smoothInputBytes.smoothRate())
+		  .detail("ActualTPS", actualTps)
+		  .detail("InputRate", inputRate)
+		  .detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate())
+		  .detail("B", b);
+		  }*/
+
+		// Don't let any storage server use up its target bytes faster than its MVCC window!
+		double maxBytesPerSecond =
+		    (targetBytes - springBytes) /
+		    ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0);
+		double limitTps = std::min(actualTps * maxBytesPerSecond / std::max(1.0e-8, inputRate),
+		                           maxBytesPerSecond * SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+		if (ssLimitReason == limitReason_t::unlimited)
+			ssLimitReason = limitReason_t::storage_server_write_bandwidth_mvcc;
+
+		if (targetRateRatio > 0 && inputRate > 0) {
+			ASSERT(inputRate != 0);
+			double smoothedRate =
+			    std::max(ss.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+			double x = smoothedRate / (inputRate * targetRateRatio);
+			double lim = actualTps * x;
+			if (lim < limitTps) {
+				double oldLimitTps = limitTps;
+				limitTps = lim;
+				if (ssLimitReason == limitReason_t::unlimited ||
+				    ssLimitReason == limitReason_t::storage_server_write_bandwidth_mvcc) {
+					if (printRateKeepLimitReasonDetails) {
+						TraceEvent("RatekeeperLimitReasonDetails")
+						    .detail("Reason", limitReason_t::storage_server_write_queue_size)
+						    .detail("FromReason", ssLimitReason)
+						    .detail("SSID", ss.id)
+						    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
+						    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
+						    .detail("LimitsStorageSpringBytes", limits->storageSpringBytes)
+						    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
+						    .detail("MinFreeSpace", minFreeSpace)
+						    .detail("SSLastReplyBytesInput", ss.lastReply.bytesInput)
+						    .detail("SSSmoothDurableBytes", ss.smoothDurableBytes.smoothTotal())
+						    .detail("StorageQueue", storageQueue)
+						    .detail("TargetBytes", targetBytes)
+						    .detail("SpringBytes", springBytes)
+						    .detail("SSVerySmoothDurableBytesSmoothRate", ss.verySmoothDurableBytes.smoothRate())
+						    .detail("SmoothedRate", smoothedRate)
+						    .detail("X", x)
+						    .detail("ActualTps", actualTps)
+						    .detail("Lim", lim)
+						    .detail("LimitTps", oldLimitTps)
+						    .detail("InputRate", inputRate)
+						    .detail("TargetRateRatio", targetRateRatio);
+					}
+					ssLimitReason = limitReason_t::storage_server_write_queue_size;
+				}
+			}
+		}
+
+		storageTpsLimitReverseIndex.insert(std::make_pair(limitTps, &ss));
+
+		if (limitTps < limits->tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space ||
+		                                    ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
+			reasonID = ss.id;
+			limits->tpsLimit = limitTps;
+			limitReason = ssLimitReason;
+		}
+
+		ssReasons[ss.id] = ssLimitReason;
+	}
+
+	std::set<Optional<Standalone<StringRef>>> ignoredMachines;
+	for (auto ss = storageTpsLimitReverseIndex.begin();
+	     ss != storageTpsLimitReverseIndex.end() && ss->first < limits->tpsLimit;
+	     ++ss) {
+		if (ignoredMachines.size() <
+		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
+			ignoredMachines.insert(ss->second->locality.zoneId());
+			continue;
+		}
+		if (ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
+			continue;
+		}
+
+		limitingStorageQueueStorageServer =
+		    ss->second->lastReply.bytesInput - ss->second->smoothDurableBytes.smoothTotal();
+		limits->tpsLimit = ss->first;
+		reasonID = storageTpsLimitReverseIndex.begin()->second->id; // Although we aren't controlling based on the worst
+		// SS, we still report it as the limiting process
+		limitReason = ssReasons[reasonID];
+		break;
+	}
+
+	// Calculate limited durability lag
+	int64_t limitingDurabilityLag = 0;
+
+	std::set<Optional<Standalone<StringRef>>> ignoredDurabilityLagMachines;
+	for (auto ss = storageDurabilityLagReverseIndex.begin(); ss != storageDurabilityLagReverseIndex.end(); ++ss) {
+		if (ignoredDurabilityLagMachines.size() <
+		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
+			ignoredDurabilityLagMachines.insert(ss->second->locality.zoneId());
+			continue;
+		}
+		if (ignoredDurabilityLagMachines.count(ss->second->locality.zoneId()) > 0) {
+			continue;
+		}
+
+		limitingDurabilityLag = -1 * ss->first;
+		if (limitingDurabilityLag > limits->durabilityLagTargetVersions &&
+		    actualTpsHistory.size() > SERVER_KNOBS->NEEDED_TPS_HISTORY_SAMPLES) {
+			if (limits->durabilityLagLimit == std::numeric_limits<double>::infinity()) {
+				double maxTps = 0;
+				for (int i = 0; i < actualTpsHistory.size(); i++) {
+					maxTps = std::max(maxTps, actualTpsHistory[i]);
+				}
+				limits->durabilityLagLimit = SERVER_KNOBS->INITIAL_DURABILITY_LAG_MULTIPLIER * maxTps;
+			}
+			if (limitingDurabilityLag > limits->lastDurabilityLag) {
+				limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_REDUCTION_RATE * limits->durabilityLagLimit;
+			}
+			if (limits->durabilityLagLimit < limits->tpsLimit) {
+				if (printRateKeepLimitReasonDetails) {
+					TraceEvent("RatekeeperLimitReasonDetails")
+					    .detail("SSID", ss->second->id)
+					    .detail("Reason", limitReason_t::storage_server_durability_lag)
+					    .detail("LimitsDurabilityLagLimit", limits->durabilityLagLimit)
+					    .detail("LimitsTpsLimit", limits->tpsLimit)
+					    .detail("LimitingDurabilityLag", limitingDurabilityLag)
+					    .detail("LimitsLastDurabilityLag", limits->lastDurabilityLag);
+				}
+				limits->tpsLimit = limits->durabilityLagLimit;
+				limitReason = limitReason_t::storage_server_durability_lag;
+			}
+		} else if (limits->durabilityLagLimit != std::numeric_limits<double>::infinity() &&
+		           limitingDurabilityLag >
+		               limits->durabilityLagTargetVersions - SERVER_KNOBS->DURABILITY_LAG_UNLIMITED_THRESHOLD) {
+			limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_INCREASE_RATE * limits->durabilityLagLimit;
+		} else {
+			limits->durabilityLagLimit = std::numeric_limits<double>::infinity();
+		}
+		limits->lastDurabilityLag = limitingDurabilityLag;
+		break;
+	}
+
+	healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
+	healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
+	healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
+	healthMetrics.limitingStorageDurabilityLag = limitingDurabilityLag;
+
+	double writeToReadLatencyLimit = 0;
+	Version worstVersionLag = 0;
+	Version limitingVersionLag = 0;
+
+	{
+		Version minSSVer = std::numeric_limits<Version>::max();
+		Version minLimitingSSVer = std::numeric_limits<Version>::max();
+		for (const auto& it : storageQueueInfo) {
+			auto& ss = it.value;
+			if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
+				continue;
+
+			minSSVer = std::min(minSSVer, ss.lastReply.version);
+
+			// Machines that ratekeeper isn't controlling can fall arbitrarily far behind
+			if (ignoredMachines.count(it.value.locality.zoneId()) == 0) {
+				minLimitingSSVer = std::min(minLimitingSSVer, ss.lastReply.version);
+			}
+		}
+
+		Version maxTLVer = std::numeric_limits<Version>::min();
+		for (const auto& it : tlogQueueInfo) {
+			auto& tl = it.value;
+			if (!tl.valid)
+				continue;
+			maxTLVer = std::max(maxTLVer, tl.lastReply.v);
+		}
+
+		if (minSSVer != std::numeric_limits<Version>::max() && maxTLVer != std::numeric_limits<Version>::min()) {
+			// writeToReadLatencyLimit: 0 = infinte speed; 1 = TL durable speed ; 2 = half TL durable speed
+			writeToReadLatencyLimit =
+			    ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference / 2) / (limits->maxVersionDifference / 4);
+			worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
+			limitingVersionLag = std::max((Version)0, maxTLVer - minLimitingSSVer);
+		}
+	}
+
+	int64_t worstFreeSpaceTLog = std::numeric_limits<int64_t>::max();
+	int64_t worstStorageQueueTLog = 0;
+	int tlcount = 0;
+	for (auto& it : tlogQueueInfo) {
+		auto& tl = it.value;
+		if (!tl.valid)
+			continue;
+		++tlcount;
+
+		limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue;
+
+		int64_t minFreeSpace =
+		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
+		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.smoothTotalSpace.smoothTotal()));
+
+		worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace);
+
+		int64_t springBytes = std::max<int64_t>(
+		    1, std::min<int64_t>(limits->logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
+		int64_t targetBytes = std::max<int64_t>(
+		    1, std::min(limits->logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
+		if (targetBytes != limits->logTargetBytes) {
+			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
+				tlogLimitReason = limitReason_t::log_server_min_free_space;
+			} else {
+				tlogLimitReason = limitReason_t::log_server_min_free_space_ratio;
+			}
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("TLogID", tl.id)
+				    .detail("Reason", tlogLimitReason)
+				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+				    .detail("TargetBytes", targetBytes)
+				    .detail("MinFreeSpace", minFreeSpace);
+			}
+		}
+
+		int64_t queue = tl.lastReply.bytesInput - tl.smoothDurableBytes.smoothTotal();
+		healthMetrics.tLogQueue[tl.id] = queue;
+		int64_t b = queue - targetBytes;
+		worstStorageQueueTLog = std::max(worstStorageQueueTLog, queue);
+
+		if (tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2) {
+			if (now() - lastWarning > 5.0) {
+				lastWarning = now();
+				TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero", id).detail("ReasonId", tl.id);
+			}
+			reasonID = tl.id;
+			limitReason = limitReason_t::log_server_min_free_space;
+			limits->tpsLimit = 0.0;
+		}
+
+		double targetRateRatio = std::min((b + springBytes) / (double)springBytes, 2.0);
+
+		if (writeToReadLatencyLimit > targetRateRatio) {
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("TLogID", tl.id)
+				    .detail("Reason", limitReason_t::storage_server_readable_behind)
+				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+				    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
+				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+				    .detail("SpringBytes", springBytes)
+				    .detail("TargetBytes", targetBytes)
+				    .detail("TLLastReplyBytesInput", tl.lastReply.bytesInput)
+				    .detail("TLSmoothDurableBytes", tl.smoothDurableBytes.smoothTotal())
+				    .detail("Queue", queue)
+				    .detail("B", b)
+				    .detail("TargetRateRatio", targetRateRatio)
+				    .detail("WriteToReadLatencyLimit", writeToReadLatencyLimit)
+				    .detail("MinFreeSpace", minFreeSpace)
+				    .detail("LimitsMaxVersionDifference", limits->maxVersionDifference);
+			}
+			targetRateRatio = writeToReadLatencyLimit;
+			tlogLimitReason = limitReason_t::storage_server_readable_behind;
+		}
+
+		double inputRate = tl.smoothInputBytes.smoothRate();
+
+		if (targetRateRatio > 0) {
+			double smoothedRate =
+			    std::max(tl.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+			double x = smoothedRate / (inputRate * targetRateRatio);
+			if (targetRateRatio < .75) //< FIXME: KNOB for 2.0
+				x = std::max(x, 0.95);
+			double lim = actualTps * x;
+			if (lim < limits->tpsLimit) {
+				limits->tpsLimit = lim;
+				reasonID = tl.id;
+				limitReason = tlogLimitReason;
+			}
+		}
+		if (inputRate > 0) {
+			// Don't let any tlogs use up its target bytes faster than its MVCC window!
+			double x =
+			    ((targetBytes - springBytes) /
+			     ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+			      2.0)) /
+			    inputRate;
+			double lim = actualTps * x;
+			if (lim < limits->tpsLimit) {
+				if (printRateKeepLimitReasonDetails) {
+					TraceEvent("RatekeeperLimitReasonDetails")
+					    .detail("Reason", limitReason_t::log_server_mvcc_write_bandwidth)
+					    .detail("TLogID", tl.id)
+					    .detail("MinFreeSpace", minFreeSpace)
+					    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+					    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+					    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
+					    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+					    .detail("SpringBytes", springBytes)
+					    .detail("TargetBytes", targetBytes)
+					    .detail("InputRate", inputRate)
+					    .detail("X", x)
+					    .detail("ActualTps", actualTps)
+					    .detail("Lim", lim)
+					    .detail("LimitsTpsLimit", limits->tpsLimit);
+				}
+				limits->tpsLimit = lim;
+				reasonID = tl.id;
+				limitReason = limitReason_t::log_server_mvcc_write_bandwidth;
+			}
+		}
+	}
+
+	healthMetrics.worstTLogQueue = worstStorageQueueTLog;
+
+	limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
+
+	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+		limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
+	}
+
+	int64_t totalDiskUsageBytes = 0;
+	for (auto& t : tlogQueueInfo) {
+		if (t.value.valid) {
+			totalDiskUsageBytes += t.value.lastReply.storageBytes.used;
+		}
+	}
+	for (auto& s : storageQueueInfo) {
+		if (s.value.valid) {
+			totalDiskUsageBytes += s.value.lastReply.storageBytes.used;
+		}
+	}
+
+	if (now() - lastSSListFetchedTimestamp > SERVER_KNOBS->STORAGE_SERVER_LIST_FETCH_TIMEOUT) {
+		limits->tpsLimit = 0.0;
+		limitReason = limitReason_t::storage_server_list_fetch_failed;
+		reasonID = UID();
+		TraceEvent(SevWarnAlways, "RkSSListFetchTimeout", id).suppressFor(1.0);
+	} else if (limits->tpsLimit == std::numeric_limits<double>::infinity()) {
+		limits->tpsLimit = SERVER_KNOBS->RATEKEEPER_DEFAULT_LIMIT;
+	}
+
+	limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
+	limits->reasonMetric = limitReason;
+
+	if (deterministicRandom()->random01() < 0.1) {
+		const std::string& name = limits->rkUpdateEventCacheHolder.getPtr()->trackingKey;
+		TraceEvent(name.c_str(), id)
+		    .detail("TPSLimit", limits->tpsLimit)
+		    .detail("Reason", limitReason)
+		    .detail("ReasonServerID", reasonID == UID() ? std::string() : Traceable<UID>::toString(reasonID))
+		    .detail("ReleasedTPS", smoothReleasedTransactions.smoothRate())
+		    .detail("ReleasedBatchTPS", smoothBatchReleasedTransactions.smoothRate())
+		    .detail("TPSBasis", actualTps)
+		    .detail("StorageServers", sscount)
+		    .detail("GrvProxies", grvProxyInfo.size())
+		    .detail("TLogs", tlcount)
+		    .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer)
+		    .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog)
+		    .detail("WorstStorageServerQueue", worstStorageQueueStorageServer)
+		    .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer)
+		    .detail("WorstTLogQueue", worstStorageQueueTLog)
+		    .detail("TotalDiskUsageBytes", totalDiskUsageBytes)
+		    .detail("WorstStorageServerVersionLag", worstVersionLag)
+		    .detail("LimitingStorageServerVersionLag", limitingVersionLag)
+		    .detail("WorstStorageServerDurabilityLag", worstDurabilityLag)
+		    .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag)
+		    .detail("TagsAutoThrottled", throttledTags.autoThrottleCount())
+		    .detail("TagsAutoThrottledBusyRead", throttledTags.busyReadTagCount)
+		    .detail("TagsAutoThrottledBusyWrite", throttledTags.busyWriteTagCount)
+		    .detail("TagsManuallyThrottled", throttledTags.manualThrottleCount())
+		    .detail("AutoThrottlingEnabled", autoThrottlingEnabled)
+		    .trackLatest(name);
+	}
+}
+
+Future<Void> RatekeeperData::refreshStorageServerCommitCost() {
+	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
+		lastBusiestCommitTagPick = now();
+		return Void();
+	}
+	double elapsed = now() - lastBusiestCommitTagPick;
+	// for each SS, select the busiest commit tag from ssTrTagCommitCost
+	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
+		it->value.busiestWriteTag.reset();
+		TransactionTag busiestTag;
+		TransactionCommitCostEstimation maxCost;
+		double maxRate = 0, maxBusyness = 0;
+		for (const auto& [tag, cost] : it->value.tagCostEst) {
+			double rate = cost.getCostSum() / elapsed;
+			if (rate > maxRate) {
+				busiestTag = tag;
+				maxRate = rate;
+				maxCost = cost;
+			}
+		}
+		if (maxRate > SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE) {
+			it->value.busiestWriteTag = busiestTag;
+			// TraceEvent("RefreshSSCommitCost").detail("TotalWriteCost", it->value.totalWriteCost).detail("TotalWriteOps",it->value.totalWriteOps);
+			ASSERT(it->value.totalWriteCosts > 0);
+			maxBusyness = double(maxCost.getCostSum()) / it->value.totalWriteCosts;
+			it->value.busiestWriteTagFractionalBusyness = maxBusyness;
+			it->value.busiestWriteTagRate = maxRate;
+		}
+
+		TraceEvent("BusiestWriteTag", it->key)
+		    .detail("Elapsed", elapsed)
+		    .detail("Tag", printable(busiestTag))
+		    .detail("TagOps", maxCost.getOpsSum())
+		    .detail("TagCost", maxCost.getCostSum())
+		    .detail("TotalCost", it->value.totalWriteCosts)
+		    .detail("Reported", it->value.busiestWriteTag.present())
+		    .trackLatest(it->value.busiestWriteTagEventHolder->trackingKey);
+
+		// reset statistics
+		it->value.tagCostEst.clear();
+		it->value.totalWriteOps = 0;
+		it->value.totalWriteCosts = 0;
+	}
+	lastBusiestCommitTagPick = now();
+	return Void();
+}
+
+void RatekeeperData::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
+	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
+	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
+	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
+		TEST(true); // Transaction tag auto-throttled
+		Optional<double> clientRate = throttledTags.autoThrottleTag(id, tag, busyness);
+		if (clientRate.present()) {
+			TagSet tags;
+			tags.addTag(tag);
+
+			Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(db.getPtr());
+			addActor.send(ThrottleApi::throttleTags(dbRef,
+			                                        tags,
+			                                        clientRate.get(),
+			                                        SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+			                                        TagThrottleType::AUTO,
+			                                        TransactionPriority::DEFAULT,
+			                                        now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+			                                        reason));
+		}
+	}
+}
+
+void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
+	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
+	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
+	// future
+	if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
+	    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
+		if (ss.busiestWriteTag.present()) {
+			tryAutoThrottleTag(ss.busiestWriteTag.get(),
+			                   ss.busiestWriteTagRate,
+			                   ss.busiestWriteTagFractionalBusyness,
+			                   TagThrottledReason::BUSY_WRITE);
+		}
+		if (ss.busiestReadTag.present()) {
+			tryAutoThrottleTag(ss.busiestReadTag.get(),
+			                   ss.busiestReadTagRate,
+			                   ss.busiestReadTagFractionalBusyness,
+			                   TagThrottledReason::BUSY_READ);
+		}
+	}
+}
diff --git a/fdbserver/RatekeeperData.h b/fdbserver/RatekeeperData.h
new file mode 100644
index 0000000000..1247e03a0e
--- /dev/null
+++ b/fdbserver/RatekeeperData.h
@@ -0,0 +1,568 @@
+/*
+ * RatekeeperData.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// TODO: Trim down these includes
+#include "fdbserver/WorkerInterface.actor.h"
+#include "flow/IndexedSet.h"
+#include "fdbrpc/FailureMonitor.h"
+#include "fdbrpc/Smoother.h"
+#include "fdbrpc/simulator.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/TagThrottle.actor.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/RatekeeperInterface.h"
+#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/WaitFailure.h"
+
+#pragma once
+
+enum limitReason_t {
+	unlimited, // TODO: rename to workload?
+	storage_server_write_queue_size, // 1
+	storage_server_write_bandwidth_mvcc,
+	storage_server_readable_behind,
+	log_server_mvcc_write_bandwidth,
+	log_server_write_queue, // 5
+	storage_server_min_free_space, // a storage server's normal limits are being reduced by low free space
+	storage_server_min_free_space_ratio, // a storage server's normal limits are being reduced by a low free space ratio
+	log_server_min_free_space,
+	log_server_min_free_space_ratio,
+	storage_server_durability_lag, // 10
+	storage_server_list_fetch_failed,
+	limitReason_t_end
+};
+
+struct StorageQueueInfo {
+	bool valid;
+	UID id;
+	LocalityData locality;
+	StorageQueuingMetricsReply lastReply;
+	StorageQueuingMetricsReply prevReply;
+	Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
+	Smoother smoothDurableVersion, smoothLatestVersion;
+	Smoother smoothFreeSpace;
+	Smoother smoothTotalSpace;
+	limitReason_t limitReason;
+
+	Optional<TransactionTag> busiestReadTag, busiestWriteTag;
+	double busiestReadTagFractionalBusyness = 0, busiestWriteTagFractionalBusyness = 0;
+	double busiestReadTagRate = 0, busiestWriteTagRate = 0;
+
+	Reference<EventCacheHolder> busiestWriteTagEventHolder;
+
+	// refresh periodically
+	TransactionTagMap<TransactionCommitCostEstimation> tagCostEst;
+	uint64_t totalWriteCosts = 0;
+	int totalWriteOps = 0;
+
+	StorageQueueInfo(UID id, LocalityData locality)
+	  : valid(false), id(id), locality(locality), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
+	    smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
+	    smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
+	    smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
+	    limitReason(limitReason_t::unlimited),
+	    busiestWriteTagEventHolder(makeReference<EventCacheHolder>(id.toString() + "/BusiestWriteTag")) {
+		// FIXME: this is a tacky workaround for a potential uninitialized use in trackStorageServerQueueInfo
+		lastReply.instanceID = -1;
+	}
+};
+
+struct TLogQueueInfo {
+	bool valid;
+	UID id;
+	TLogQueuingMetricsReply lastReply;
+	TLogQueuingMetricsReply prevReply;
+	Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
+	Smoother smoothFreeSpace;
+	Smoother smoothTotalSpace;
+	TLogQueueInfo(UID id)
+	  : valid(false), id(id), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
+	    smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
+	    smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT) {
+		// FIXME: this is a tacky workaround for a potential uninitialized use in trackTLogQueueInfo (copied from
+		// storageQueueInfO)
+		lastReply.instanceID = -1;
+	}
+};
+
+class RkTagThrottleCollection : NonCopyable {
+private:
+	struct RkTagData {
+		Smoother requestRate;
+		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+	};
+
+	struct RkTagThrottleData {
+		ClientTagThrottleLimits limits;
+		Smoother clientRate;
+
+		// Only used by auto-throttles
+		double created = now();
+		double lastUpdated = 0;
+		double lastReduced = now();
+		bool rateSet = false;
+
+		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+
+		double getTargetRate(Optional<double> requestRate) {
+			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
+				return limits.tpsRate;
+			} else {
+				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
+			}
+		}
+
+		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
+			if (limits.expiration > now()) {
+				double targetRate = getTargetRate(requestRate);
+				if (targetRate == std::numeric_limits<double>::max()) {
+					rateSet = false;
+					return targetRate;
+				}
+				if (!rateSet) {
+					rateSet = true;
+					clientRate.reset(targetRate);
+				} else {
+					clientRate.setTotal(targetRate);
+				}
+
+				double rate = clientRate.smoothTotal();
+				ASSERT(rate >= 0);
+				return rate;
+			} else {
+				TEST(true); // Get throttle rate for expired throttle
+				rateSet = false;
+				return Optional<double>();
+			}
+		}
+	};
+
+	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
+
+public:
+	RkTagThrottleCollection() {}
+
+	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	void operator=(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
+		ASSERT(currentBusyness > 0);
+
+		if (targetBusyness < 1) {
+			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
+			return requestRate * targetFraction;
+		} else {
+			return std::numeric_limits<double>::max();
+		}
+	}
+
+	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
+	Optional<double> autoThrottleTag(UID id,
+	                                 TransactionTag const& tag,
+	                                 double fractionalBusyness,
+	                                 Optional<double> tpsRate = Optional<double>(),
+	                                 Optional<double> expiration = Optional<double>()) {
+		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
+		ASSERT(!expiration.present() || expiration.get() > now());
+
+		auto itr = autoThrottledTags.find(tag);
+		bool present = (itr != autoThrottledTags.end());
+		if (!present) {
+			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
+				TEST(true); // Reached auto-throttle limit
+				return Optional<double>();
+			}
+
+			itr = autoThrottledTags.try_emplace(tag).first;
+			initializeTag(tag);
+		} else if (itr->second.limits.expiration <= now()) {
+			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
+			present = false;
+			itr->second = RkTagThrottleData();
+		}
+
+		auto& throttle = itr->second;
+
+		if (!tpsRate.present()) {
+			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
+				tpsRate = std::numeric_limits<double>::max();
+				if (present) {
+					return Optional<double>();
+				}
+			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
+				TEST(true); // Tag auto-throttled too quickly
+				return Optional<double>();
+			} else {
+				tpsRate = computeTargetTpsRate(fractionalBusyness,
+				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
+				                               tagData[tag].requestRate.smoothRate());
+
+				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
+					TEST(true); // Tag auto-throttle rate increase attempt while active
+					return Optional<double>();
+				}
+
+				throttle.lastUpdated = now();
+				if (tpsRate.get() < throttle.limits.tpsRate) {
+					throttle.lastReduced = now();
+				}
+			}
+		}
+		if (!expiration.present()) {
+			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
+		}
+
+		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
+
+		throttle.limits.tpsRate = tpsRate.get();
+		throttle.limits.expiration = expiration.get();
+
+		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
+
+		TraceEvent("RkSetAutoThrottle", id)
+		    .detail("Tag", tag)
+		    .detail("TargetRate", tpsRate.get())
+		    .detail("Expiration", expiration.get() - now())
+		    .detail("ClientRate", clientRate)
+		    .detail("Created", now() - throttle.created)
+		    .detail("LastUpdate", now() - throttle.lastUpdated)
+		    .detail("LastReduced", now() - throttle.lastReduced);
+
+		if (tpsRate.get() != std::numeric_limits<double>::max()) {
+			return tpsRate.get();
+		} else {
+			return Optional<double>();
+		}
+	}
+
+	void manualThrottleTag(UID id,
+	                       TransactionTag const& tag,
+	                       TransactionPriority priority,
+	                       double tpsRate,
+	                       double expiration,
+	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
+		ASSERT(tpsRate >= 0);
+		ASSERT(expiration > now());
+
+		auto& priorityThrottleMap = manualThrottledTags[tag];
+		auto result = priorityThrottleMap.try_emplace(priority);
+		initializeTag(tag);
+		ASSERT(result.second); // Updating to the map is done by copying the whole map
+
+		result.first->second.limits.tpsRate = tpsRate;
+		result.first->second.limits.expiration = expiration;
+
+		if (!oldLimits.present()) {
+			TEST(true); // Transaction tag manually throttled
+			TraceEvent("RatekeeperAddingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
+			TEST(true); // Manual transaction tag throttle updated
+			TraceEvent("RatekeeperUpdatingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		}
+
+		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
+		ASSERT(clientRate.present());
+	}
+
+	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
+	                                                             TransactionPriority priority) {
+		auto itr = manualThrottledTags.find(tag);
+		if (itr != manualThrottledTags.end()) {
+			auto priorityItr = itr->second.find(priority);
+			if (priorityItr != itr->second.end()) {
+				return priorityItr->second.limits;
+			}
+		}
+
+		return Optional<ClientTagThrottleLimits>();
+	}
+
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
+		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
+
+		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
+			bool tagPresent = false;
+
+			double requestRate = tagItr->second.requestRate.smoothRate();
+			auto manualItr = manualThrottledTags.find(tagItr->first);
+			if (manualItr != manualThrottledTags.end()) {
+				Optional<ClientTagThrottleLimits> manualClientRate;
+				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
+				     ++priority) {
+					auto priorityItr = manualItr->second.find(*priority);
+					if (priorityItr != manualItr->second.end()) {
+						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
+						if (!priorityClientRate.present()) {
+							TEST(true); // Manual priority throttle expired
+							priorityItr = manualItr->second.erase(priorityItr);
+						} else {
+							if (!manualClientRate.present() ||
+							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
+								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
+								                                           priorityItr->second.limits.expiration);
+							} else {
+								TEST(true); // Manual throttle overriden by higher priority
+							}
+
+							++priorityItr;
+						}
+					}
+
+					if (manualClientRate.present()) {
+						tagPresent = true;
+						TEST(true); // Using manual throttle
+						clientRates[*priority][tagItr->first] = manualClientRate.get();
+					}
+				}
+
+				if (manualItr->second.empty()) {
+					TEST(true); // All manual throttles expired
+					manualThrottledTags.erase(manualItr);
+					break;
+				}
+			}
+
+			auto autoItr = autoThrottledTags.find(tagItr->first);
+			if (autoItr != autoThrottledTags.end()) {
+				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
+				if (autoClientRate.present()) {
+					double adjustedRate = autoClientRate.get();
+					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
+					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
+						TEST(true); // Tag auto-throttle ramping up
+
+						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
+						if (targetBusyness == 0) {
+							targetBusyness = 0.01;
+						}
+
+						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+						adjustedRate =
+						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
+					}
+
+					tagPresent = true;
+					if (autoThrottlingEnabled) {
+						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
+						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
+						if (!result.second && result.first->second.tpsRate > adjustedRate) {
+							result.first->second =
+							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
+						} else {
+							TEST(true); // Auto throttle overriden by manual throttle
+						}
+						clientRates[TransactionPriority::BATCH][tagItr->first] =
+						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
+					}
+				} else {
+					ASSERT(autoItr->second.limits.expiration <= now());
+					TEST(true); // Auto throttle expired
+					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
+						tagPresent = true;
+					} else {
+						autoThrottledTags.erase(autoItr);
+					}
+				}
+			}
+
+			if (!tagPresent) {
+				TEST(true); // All tag throttles expired
+				tagItr = tagData.erase(tagItr);
+			} else {
+				++tagItr;
+			}
+		}
+
+		return clientRates;
+	}
+
+	void addRequests(TransactionTag const& tag, int requests) {
+		if (requests > 0) {
+			TEST(true); // Requests reported for throttled tag
+
+			auto tagItr = tagData.try_emplace(tag);
+			tagItr.first->second.requestRate.addDelta(requests);
+
+			double requestRate = tagItr.first->second.requestRate.smoothRate();
+
+			auto autoItr = autoThrottledTags.find(tag);
+			if (autoItr != autoThrottledTags.end()) {
+				autoItr->second.updateAndGetClientRate(requestRate);
+			}
+
+			auto manualItr = manualThrottledTags.find(tag);
+			if (manualItr != manualThrottledTags.end()) {
+				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
+				     ++priorityItr) {
+					priorityItr->second.updateAndGetClientRate(requestRate);
+				}
+			}
+		}
+	}
+
+	Optional<double> getRequestRate(TransactionTag const& tag) {
+		auto itr = tagData.find(tag);
+		if (itr != tagData.end()) {
+			return itr->second.requestRate.smoothRate();
+		}
+		return Optional<double>();
+	}
+
+	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
+
+	int64_t manualThrottleCount() const {
+		int64_t count = 0;
+		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
+			count += itr->second.size();
+		}
+
+		return count;
+	}
+
+	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
+	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
+	TransactionTagMap<RkTagData> tagData;
+	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
+};
+
+struct RatekeeperLimits {
+	double tpsLimit;
+	Int64MetricHandle tpsLimitMetric;
+	Int64MetricHandle reasonMetric;
+
+	int64_t storageTargetBytes;
+	int64_t storageSpringBytes;
+	int64_t logTargetBytes;
+	int64_t logSpringBytes;
+	double maxVersionDifference;
+
+	int64_t durabilityLagTargetVersions;
+	int64_t lastDurabilityLag;
+	double durabilityLagLimit;
+
+	TransactionPriority priority;
+	std::string context;
+
+	Reference<EventCacheHolder> rkUpdateEventCacheHolder;
+
+	RatekeeperLimits(TransactionPriority priority,
+	                 std::string context,
+	                 int64_t storageTargetBytes,
+	                 int64_t storageSpringBytes,
+	                 int64_t logTargetBytes,
+	                 int64_t logSpringBytes,
+	                 double maxVersionDifference,
+	                 int64_t durabilityLagTargetVersions)
+	  : tpsLimit(std::numeric_limits<double>::infinity()), tpsLimitMetric(StringRef("Ratekeeper.TPSLimit" + context)),
+	    reasonMetric(StringRef("Ratekeeper.Reason" + context)), storageTargetBytes(storageTargetBytes),
+	    storageSpringBytes(storageSpringBytes), logTargetBytes(logTargetBytes), logSpringBytes(logSpringBytes),
+	    maxVersionDifference(maxVersionDifference),
+	    durabilityLagTargetVersions(
+	        durabilityLagTargetVersions +
+	        SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions are expected to not
+	                                                           // be durable on the storage servers
+	    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), priority(priority),
+	    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
+};
+
+struct RatekeeperData {
+	// Differentiate from GrvProxyInfo in DatabaseContext.h
+	struct GrvProxyInfo {
+		int64_t totalTransactions;
+		int64_t batchTransactions;
+		uint64_t lastThrottledTagChangeId;
+
+		double lastUpdateTime;
+		double lastTagPushTime;
+
+		GrvProxyInfo()
+		  : totalTransactions(0), batchTransactions(0), lastThrottledTagChangeId(0), lastUpdateTime(0),
+		    lastTagPushTime(0) {}
+	};
+
+	UID id;
+	Database db;
+
+	Map<UID, StorageQueueInfo> storageQueueInfo;
+	Map<UID, TLogQueueInfo> tlogQueueInfo;
+
+	std::map<UID, RatekeeperData::GrvProxyInfo> grvProxyInfo;
+	Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes;
+	HealthMetrics healthMetrics;
+	DatabaseConfiguration configuration;
+	PromiseStream<Future<Void>> addActor;
+
+	Int64MetricHandle actualTpsMetric;
+
+	double lastWarning;
+	double lastSSListFetchedTimestamp;
+	double lastBusiestCommitTagPick;
+
+	RkTagThrottleCollection throttledTags;
+	uint64_t throttledTagChangeId;
+
+	RatekeeperLimits normalLimits;
+	RatekeeperLimits batchLimits;
+
+	Deque<double> actualTpsHistory;
+	Optional<Key> remoteDC;
+
+	Future<Void> expiredTagThrottleCleanup;
+
+	bool autoThrottlingEnabled;
+
+	RatekeeperData(UID id, Database db);
+
+	Future<Void> configurationMonitor();
+	void updateCommitCostEstimation(UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation);
+	void updateRate(RatekeeperLimits* limits);
+	Future<Void> refreshStorageServerCommitCost();
+	Future<Void> monitorServerListChange(PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
+	Future<Void> trackEachStorageServer(FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
+
+	// SOMEDAY: template trackStorageServerQueueInfo and trackTLogQueueInfo into one function
+	Future<Void> trackStorageServerQueueInfo(StorageServerInterface);
+	Future<Void> trackTLogQueueInfo(TLogInterface);
+
+	void tryAutoThrottleTag(TransactionTag, double rate, double busyness, TagThrottledReason);
+	void tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
+
+	static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
+};

From 687df447ce44d18f104d5ad92df14b5c8e50cb39 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 12:32:34 -0800
Subject: [PATCH 060/210] Move all code from Ratekeeper.actor.cpp into
 RatekeeperData.actor.cpp

---
 fdbserver/Ratekeeper.actor.cpp     | 1618 +---------------------------
 fdbserver/RatekeeperData.actor.cpp |  298 +++++
 fdbserver/RatekeeperData.h         |    1 +
 3 files changed, 300 insertions(+), 1617 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 079b70eae4..76d8939a11 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -28,1624 +28,8 @@
 #include "fdbclient/TagThrottle.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/RatekeeperData.h"
 #include "fdbserver/RatekeeperInterface.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbserver/WaitFailure.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
-
-enum limitReason_t {
-	unlimited, // TODO: rename to workload?
-	storage_server_write_queue_size, // 1
-	storage_server_write_bandwidth_mvcc,
-	storage_server_readable_behind,
-	log_server_mvcc_write_bandwidth,
-	log_server_write_queue, // 5
-	storage_server_min_free_space, // a storage server's normal limits are being reduced by low free space
-	storage_server_min_free_space_ratio, // a storage server's normal limits are being reduced by a low free space ratio
-	log_server_min_free_space,
-	log_server_min_free_space_ratio,
-	storage_server_durability_lag, // 10
-	storage_server_list_fetch_failed,
-	limitReason_t_end
-};
-
-int limitReasonEnd = limitReason_t_end;
-
-const char* limitReasonName[] = { "workload",
-	                              "storage_server_write_queue_size",
-	                              "storage_server_write_bandwidth_mvcc",
-	                              "storage_server_readable_behind",
-	                              "log_server_mvcc_write_bandwidth",
-	                              "log_server_write_queue",
-	                              "storage_server_min_free_space",
-	                              "storage_server_min_free_space_ratio",
-	                              "log_server_min_free_space",
-	                              "log_server_min_free_space_ratio",
-	                              "storage_server_durability_lag",
-	                              "storage_server_list_fetch_failed" };
-static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-// NOTE: This has a corresponding table in Script.cs (see RatekeeperReason graph)
-// IF UPDATING THIS ARRAY, UPDATE SCRIPT.CS!
-const char* limitReasonDesc[] = { "Workload or read performance.",
-	                              "Storage server performance (storage queue).",
-	                              "Storage server MVCC memory.",
-	                              "Storage server version falling behind.",
-	                              "Log server MVCC memory.",
-	                              "Storage server performance (log queue).",
-	                              "Storage server running out of space (approaching 100MB limit).",
-	                              "Storage server running out of space (approaching 5% limit).",
-	                              "Log server running out of space (approaching 100MB limit).",
-	                              "Log server running out of space (approaching 5% limit).",
-	                              "Storage server durable version falling behind.",
-	                              "Unable to fetch storage server list." };
-
-static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-struct StorageQueueInfo {
-	bool valid;
-	UID id;
-	LocalityData locality;
-	StorageQueuingMetricsReply lastReply;
-	StorageQueuingMetricsReply prevReply;
-	Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
-	Smoother smoothDurableVersion, smoothLatestVersion;
-	Smoother smoothFreeSpace;
-	Smoother smoothTotalSpace;
-	limitReason_t limitReason;
-
-	Optional<TransactionTag> busiestReadTag, busiestWriteTag;
-	double busiestReadTagFractionalBusyness = 0, busiestWriteTagFractionalBusyness = 0;
-	double busiestReadTagRate = 0, busiestWriteTagRate = 0;
-
-	Reference<EventCacheHolder> busiestWriteTagEventHolder;
-
-	// refresh periodically
-	TransactionTagMap<TransactionCommitCostEstimation> tagCostEst;
-	uint64_t totalWriteCosts = 0;
-	int totalWriteOps = 0;
-
-	StorageQueueInfo(UID id, LocalityData locality)
-	  : valid(false), id(id), locality(locality), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-	    smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    limitReason(limitReason_t::unlimited),
-	    busiestWriteTagEventHolder(makeReference<EventCacheHolder>(id.toString() + "/BusiestWriteTag")) {
-		// FIXME: this is a tacky workaround for a potential uninitialized use in trackStorageServerQueueInfo
-		lastReply.instanceID = -1;
-	}
-};
-
-struct TLogQueueInfo {
-	bool valid;
-	UID id;
-	TLogQueuingMetricsReply lastReply;
-	TLogQueuingMetricsReply prevReply;
-	Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
-	Smoother smoothFreeSpace;
-	Smoother smoothTotalSpace;
-	TLogQueueInfo(UID id)
-	  : valid(false), id(id), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-	    smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT) {
-		// FIXME: this is a tacky workaround for a potential uninitialized use in trackTLogQueueInfo (copied from
-		// storageQueueInfO)
-		lastReply.instanceID = -1;
-	}
-};
-
-class RkTagThrottleCollection : NonCopyable {
-private:
-	struct RkTagData {
-		Smoother requestRate;
-		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-	};
-
-	struct RkTagThrottleData {
-		ClientTagThrottleLimits limits;
-		Smoother clientRate;
-
-		// Only used by auto-throttles
-		double created = now();
-		double lastUpdated = 0;
-		double lastReduced = now();
-		bool rateSet = false;
-
-		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-
-		double getTargetRate(Optional<double> requestRate) {
-			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
-				return limits.tpsRate;
-			} else {
-				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
-			}
-		}
-
-		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
-			if (limits.expiration > now()) {
-				double targetRate = getTargetRate(requestRate);
-				if (targetRate == std::numeric_limits<double>::max()) {
-					rateSet = false;
-					return targetRate;
-				}
-				if (!rateSet) {
-					rateSet = true;
-					clientRate.reset(targetRate);
-				} else {
-					clientRate.setTotal(targetRate);
-				}
-
-				double rate = clientRate.smoothTotal();
-				ASSERT(rate >= 0);
-				return rate;
-			} else {
-				TEST(true); // Get throttle rate for expired throttle
-				rateSet = false;
-				return Optional<double>();
-			}
-		}
-	};
-
-	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
-
-public:
-	RkTagThrottleCollection() {}
-
-	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	void operator=(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
-		ASSERT(currentBusyness > 0);
-
-		if (targetBusyness < 1) {
-			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
-			return requestRate * targetFraction;
-		} else {
-			return std::numeric_limits<double>::max();
-		}
-	}
-
-	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
-	Optional<double> autoThrottleTag(UID id,
-	                                 TransactionTag const& tag,
-	                                 double fractionalBusyness,
-	                                 Optional<double> tpsRate = Optional<double>(),
-	                                 Optional<double> expiration = Optional<double>()) {
-		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
-		ASSERT(!expiration.present() || expiration.get() > now());
-
-		auto itr = autoThrottledTags.find(tag);
-		bool present = (itr != autoThrottledTags.end());
-		if (!present) {
-			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
-				TEST(true); // Reached auto-throttle limit
-				return Optional<double>();
-			}
-
-			itr = autoThrottledTags.try_emplace(tag).first;
-			initializeTag(tag);
-		} else if (itr->second.limits.expiration <= now()) {
-			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
-			present = false;
-			itr->second = RkTagThrottleData();
-		}
-
-		auto& throttle = itr->second;
-
-		if (!tpsRate.present()) {
-			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
-				tpsRate = std::numeric_limits<double>::max();
-				if (present) {
-					return Optional<double>();
-				}
-			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
-				TEST(true); // Tag auto-throttled too quickly
-				return Optional<double>();
-			} else {
-				tpsRate = computeTargetTpsRate(fractionalBusyness,
-				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
-				                               tagData[tag].requestRate.smoothRate());
-
-				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
-					TEST(true); // Tag auto-throttle rate increase attempt while active
-					return Optional<double>();
-				}
-
-				throttle.lastUpdated = now();
-				if (tpsRate.get() < throttle.limits.tpsRate) {
-					throttle.lastReduced = now();
-				}
-			}
-		}
-		if (!expiration.present()) {
-			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
-		}
-
-		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
-
-		throttle.limits.tpsRate = tpsRate.get();
-		throttle.limits.expiration = expiration.get();
-
-		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
-
-		TraceEvent("RkSetAutoThrottle", id)
-		    .detail("Tag", tag)
-		    .detail("TargetRate", tpsRate.get())
-		    .detail("Expiration", expiration.get() - now())
-		    .detail("ClientRate", clientRate)
-		    .detail("Created", now() - throttle.created)
-		    .detail("LastUpdate", now() - throttle.lastUpdated)
-		    .detail("LastReduced", now() - throttle.lastReduced);
-
-		if (tpsRate.get() != std::numeric_limits<double>::max()) {
-			return tpsRate.get();
-		} else {
-			return Optional<double>();
-		}
-	}
-
-	void manualThrottleTag(UID id,
-	                       TransactionTag const& tag,
-	                       TransactionPriority priority,
-	                       double tpsRate,
-	                       double expiration,
-	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
-		ASSERT(tpsRate >= 0);
-		ASSERT(expiration > now());
-
-		auto& priorityThrottleMap = manualThrottledTags[tag];
-		auto result = priorityThrottleMap.try_emplace(priority);
-		initializeTag(tag);
-		ASSERT(result.second); // Updating to the map is done by copying the whole map
-
-		result.first->second.limits.tpsRate = tpsRate;
-		result.first->second.limits.expiration = expiration;
-
-		if (!oldLimits.present()) {
-			TEST(true); // Transaction tag manually throttled
-			TraceEvent("RatekeeperAddingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
-			TEST(true); // Manual transaction tag throttle updated
-			TraceEvent("RatekeeperUpdatingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		}
-
-		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
-		ASSERT(clientRate.present());
-	}
-
-	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
-	                                                             TransactionPriority priority) {
-		auto itr = manualThrottledTags.find(tag);
-		if (itr != manualThrottledTags.end()) {
-			auto priorityItr = itr->second.find(priority);
-			if (priorityItr != itr->second.end()) {
-				return priorityItr->second.limits;
-			}
-		}
-
-		return Optional<ClientTagThrottleLimits>();
-	}
-
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
-		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
-
-		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
-			bool tagPresent = false;
-
-			double requestRate = tagItr->second.requestRate.smoothRate();
-			auto manualItr = manualThrottledTags.find(tagItr->first);
-			if (manualItr != manualThrottledTags.end()) {
-				Optional<ClientTagThrottleLimits> manualClientRate;
-				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
-				     ++priority) {
-					auto priorityItr = manualItr->second.find(*priority);
-					if (priorityItr != manualItr->second.end()) {
-						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
-						if (!priorityClientRate.present()) {
-							TEST(true); // Manual priority throttle expired
-							priorityItr = manualItr->second.erase(priorityItr);
-						} else {
-							if (!manualClientRate.present() ||
-							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
-								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
-								                                           priorityItr->second.limits.expiration);
-							} else {
-								TEST(true); // Manual throttle overriden by higher priority
-							}
-
-							++priorityItr;
-						}
-					}
-
-					if (manualClientRate.present()) {
-						tagPresent = true;
-						TEST(true); // Using manual throttle
-						clientRates[*priority][tagItr->first] = manualClientRate.get();
-					}
-				}
-
-				if (manualItr->second.empty()) {
-					TEST(true); // All manual throttles expired
-					manualThrottledTags.erase(manualItr);
-					break;
-				}
-			}
-
-			auto autoItr = autoThrottledTags.find(tagItr->first);
-			if (autoItr != autoThrottledTags.end()) {
-				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
-				if (autoClientRate.present()) {
-					double adjustedRate = autoClientRate.get();
-					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
-					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
-						TEST(true); // Tag auto-throttle ramping up
-
-						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
-						if (targetBusyness == 0) {
-							targetBusyness = 0.01;
-						}
-
-						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-						adjustedRate =
-						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
-					}
-
-					tagPresent = true;
-					if (autoThrottlingEnabled) {
-						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
-						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
-						if (!result.second && result.first->second.tpsRate > adjustedRate) {
-							result.first->second =
-							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
-						} else {
-							TEST(true); // Auto throttle overriden by manual throttle
-						}
-						clientRates[TransactionPriority::BATCH][tagItr->first] =
-						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
-					}
-				} else {
-					ASSERT(autoItr->second.limits.expiration <= now());
-					TEST(true); // Auto throttle expired
-					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
-						tagPresent = true;
-					} else {
-						autoThrottledTags.erase(autoItr);
-					}
-				}
-			}
-
-			if (!tagPresent) {
-				TEST(true); // All tag throttles expired
-				tagItr = tagData.erase(tagItr);
-			} else {
-				++tagItr;
-			}
-		}
-
-		return clientRates;
-	}
-
-	void addRequests(TransactionTag const& tag, int requests) {
-		if (requests > 0) {
-			TEST(true); // Requests reported for throttled tag
-
-			auto tagItr = tagData.try_emplace(tag);
-			tagItr.first->second.requestRate.addDelta(requests);
-
-			double requestRate = tagItr.first->second.requestRate.smoothRate();
-
-			auto autoItr = autoThrottledTags.find(tag);
-			if (autoItr != autoThrottledTags.end()) {
-				autoItr->second.updateAndGetClientRate(requestRate);
-			}
-
-			auto manualItr = manualThrottledTags.find(tag);
-			if (manualItr != manualThrottledTags.end()) {
-				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
-				     ++priorityItr) {
-					priorityItr->second.updateAndGetClientRate(requestRate);
-				}
-			}
-		}
-	}
-
-	Optional<double> getRequestRate(TransactionTag const& tag) {
-		auto itr = tagData.find(tag);
-		if (itr != tagData.end()) {
-			return itr->second.requestRate.smoothRate();
-		}
-		return Optional<double>();
-	}
-
-	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
-
-	int64_t manualThrottleCount() const {
-		int64_t count = 0;
-		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
-			count += itr->second.size();
-		}
-
-		return count;
-	}
-
-	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
-	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
-	TransactionTagMap<RkTagData> tagData;
-	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
-};
-
-struct RatekeeperLimits {
-	double tpsLimit;
-	Int64MetricHandle tpsLimitMetric;
-	Int64MetricHandle reasonMetric;
-
-	int64_t storageTargetBytes;
-	int64_t storageSpringBytes;
-	int64_t logTargetBytes;
-	int64_t logSpringBytes;
-	double maxVersionDifference;
-
-	int64_t durabilityLagTargetVersions;
-	int64_t lastDurabilityLag;
-	double durabilityLagLimit;
-
-	TransactionPriority priority;
-	std::string context;
-
-	Reference<EventCacheHolder> rkUpdateEventCacheHolder;
-
-	RatekeeperLimits(TransactionPriority priority,
-	                 std::string context,
-	                 int64_t storageTargetBytes,
-	                 int64_t storageSpringBytes,
-	                 int64_t logTargetBytes,
-	                 int64_t logSpringBytes,
-	                 double maxVersionDifference,
-	                 int64_t durabilityLagTargetVersions)
-	  : tpsLimit(std::numeric_limits<double>::infinity()), tpsLimitMetric(StringRef("Ratekeeper.TPSLimit" + context)),
-	    reasonMetric(StringRef("Ratekeeper.Reason" + context)), storageTargetBytes(storageTargetBytes),
-	    storageSpringBytes(storageSpringBytes), logTargetBytes(logTargetBytes), logSpringBytes(logSpringBytes),
-	    maxVersionDifference(maxVersionDifference),
-	    durabilityLagTargetVersions(
-	        durabilityLagTargetVersions +
-	        SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions are expected to not
-	                                                           // be durable on the storage servers
-	    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), priority(priority),
-	    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
-};
-
-namespace RatekeeperActorCpp {
-
-// Differentiate from GrvProxyInfo in DatabaseContext.h
-struct GrvProxyInfo {
-	int64_t totalTransactions;
-	int64_t batchTransactions;
-	uint64_t lastThrottledTagChangeId;
-
-	double lastUpdateTime;
-	double lastTagPushTime;
-
-	GrvProxyInfo()
-	  : totalTransactions(0), batchTransactions(0), lastThrottledTagChangeId(0), lastUpdateTime(0), lastTagPushTime(0) {
-	}
-};
-
-} // namespace RatekeeperActorCpp
-
-struct RatekeeperData {
-	UID id;
-	Database db;
-
-	Map<UID, StorageQueueInfo> storageQueueInfo;
-	Map<UID, TLogQueueInfo> tlogQueueInfo;
-
-	std::map<UID, RatekeeperActorCpp::GrvProxyInfo> grvProxyInfo;
-	Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes;
-	HealthMetrics healthMetrics;
-	DatabaseConfiguration configuration;
-	PromiseStream<Future<Void>> addActor;
-
-	Int64MetricHandle actualTpsMetric;
-
-	double lastWarning;
-	double lastSSListFetchedTimestamp;
-	double lastBusiestCommitTagPick;
-
-	RkTagThrottleCollection throttledTags;
-	uint64_t throttledTagChangeId;
-
-	RatekeeperLimits normalLimits;
-	RatekeeperLimits batchLimits;
-
-	Deque<double> actualTpsHistory;
-	Optional<Key> remoteDC;
-
-	Future<Void> expiredTagThrottleCleanup;
-
-	bool autoThrottlingEnabled;
-
-	RatekeeperData(UID id, Database db)
-	  : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
-	    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-	    actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
-	    lastBusiestCommitTagPick(0), throttledTagChangeId(0),
-	    normalLimits(TransactionPriority::DEFAULT,
-	                 "",
-	                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
-	                 SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
-	                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
-	                 SERVER_KNOBS->SPRING_BYTES_TLOG,
-	                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
-	                 SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
-	    batchLimits(TransactionPriority::BATCH,
-	                "Batch",
-	                SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
-	                SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH,
-	                SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
-	                SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
-	                SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-	                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
-	    autoThrottlingEnabled(false) {
-		expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
-		                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
-	}
-};
-
-// SOMEDAY: template trackStorageServerQueueInfo and trackTLogQueueInfo into one function
-ACTOR Future<Void> trackStorageServerQueueInfo(RatekeeperData* self, StorageServerInterface ssi) {
-	self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
-	state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
-	TraceEvent("RkTracking", self->id).detail("StorageServer", ssi.id()).detail("Locality", ssi.locality.toString());
-	try {
-		loop {
-			ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
-			    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-			if (reply.present()) {
-				myQueueInfo->value.valid = true;
-				myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
-				myQueueInfo->value.lastReply = reply.get();
-				if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
-					myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
-					myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
-					myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
-					myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
-					myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
-					myQueueInfo->value.smoothDurableVersion.reset(reply.get().durableVersion);
-					myQueueInfo->value.smoothLatestVersion.reset(reply.get().version);
-				} else {
-					self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
-					                                       myQueueInfo->value.prevReply.bytesDurable);
-					myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
-					myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
-					myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
-					myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
-					myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
-					myQueueInfo->value.smoothDurableVersion.setTotal(reply.get().durableVersion);
-					myQueueInfo->value.smoothLatestVersion.setTotal(reply.get().version);
-				}
-
-				myQueueInfo->value.busiestReadTag = reply.get().busiestTag;
-				myQueueInfo->value.busiestReadTagFractionalBusyness = reply.get().busiestTagFractionalBusyness;
-				myQueueInfo->value.busiestReadTagRate = reply.get().busiestTagRate;
-			} else {
-				if (myQueueInfo->value.valid) {
-					TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id());
-				}
-				myQueueInfo->value.valid = false;
-			}
-
-			wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-			     IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
-			                                                    FailureStatus(false)));
-		}
-	} catch (...) {
-		// including cancellation
-		self->storageQueueInfo.erase(myQueueInfo);
-		throw;
-	}
-}
-
-ACTOR Future<Void> trackTLogQueueInfo(RatekeeperData* self, TLogInterface tli) {
-	self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
-	state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
-	TraceEvent("RkTracking", self->id).detail("TransactionLog", tli.id());
-	try {
-		loop {
-			ErrorOr<TLogQueuingMetricsReply> reply = wait(tli.getQueuingMetrics.getReplyUnlessFailedFor(
-			    TLogQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-			if (reply.present()) {
-				myQueueInfo->value.valid = true;
-				myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
-				myQueueInfo->value.lastReply = reply.get();
-				if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
-					myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
-					myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
-					myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
-					myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
-					myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
-				} else {
-					self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
-					                                       myQueueInfo->value.prevReply.bytesDurable);
-					myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
-					myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
-					myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
-					myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
-					myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
-				}
-			} else {
-				if (myQueueInfo->value.valid) {
-					TraceEvent("RkTLogDidNotRespond", self->id).detail("TransactionLog", tli.id());
-				}
-				myQueueInfo->value.valid = false;
-			}
-
-			wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-			     IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
-			                                                    FailureStatus(false)));
-		}
-	} catch (...) {
-		// including cancellation
-		self->tlogQueueInfo.erase(myQueueInfo);
-		throw;
-	}
-}
-
-ACTOR Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
-	try {
-		wait(in);
-		return Void();
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && !errOut.isSet())
-			errOut.sendError(e);
-		throw;
-	}
-}
-
-ACTOR Future<Void> trackEachStorageServer(
-    RatekeeperData* self,
-    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	state Map<UID, Future<Void>> actors;
-	state Promise<Void> err;
-	loop choose {
-		when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
-			wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
-			if (change.second.present()) {
-				if (!change.second.get().isTss()) {
-					auto& a = actors[change.first];
-					a = Future<Void>();
-					a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
-				}
-			} else
-				actors.erase(change.first);
-		}
-		when(wait(err.getFuture())) {}
-	}
-}
-
-ACTOR Future<Void> monitorServerListChange(
-    RatekeeperData* self,
-    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	state std::map<UID, StorageServerInterface> oldServers;
-	state Transaction tr(self->db);
-
-	loop {
-		try {
-			if (now() - self->lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
-				TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", self->id)
-				    .detail("Latency", now() - self->lastSSListFetchedTimestamp);
-			}
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-			    wait(getServerListAndProcessClasses(&tr));
-			self->lastSSListFetchedTimestamp = now();
-
-			std::map<UID, StorageServerInterface> newServers;
-			for (const auto& [ssi, _] : results) {
-				const UID serverId = ssi.id();
-				newServers[serverId] = ssi;
-
-				if (oldServers.count(serverId)) {
-					if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) {
-						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-					}
-					oldServers.erase(serverId);
-				} else {
-					serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-				}
-			}
-
-			for (const auto& it : oldServers) {
-				serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
-			}
-
-			oldServers.swap(newServers);
-			tr = Transaction(self->db);
-			wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
-		} catch (Error& e) {
-			TraceEvent("RatekeeperGetSSListError", self->id).error(e).suppressFor(1.0);
-			wait(tr.onError(e));
-		}
-	}
-}
-
-ACTOR Future<Void> monitorThrottlingChanges(RatekeeperData* self) {
-	state bool committed = false;
-	loop {
-		state ReadYourWritesTransaction tr(self->db);
-
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-
-				state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
-
-				if (!committed) {
-					BinaryWriter limitWriter(Unversioned());
-					limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
-					tr.set(tagThrottleLimitKey, limitWriter.toValue());
-				}
-
-				wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
-
-				if (autoThrottlingEnabled.get().present() &&
-				    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
-					TEST(true); // Auto-throttling disabled
-					if (self->autoThrottlingEnabled) {
-						TraceEvent("AutoTagThrottlingDisabled", self->id).log();
-					}
-					self->autoThrottlingEnabled = false;
-				} else if (autoThrottlingEnabled.get().present() &&
-				           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
-					TEST(true); // Auto-throttling enabled
-					if (!self->autoThrottlingEnabled) {
-						TraceEvent("AutoTagThrottlingEnabled", self->id).log();
-					}
-					self->autoThrottlingEnabled = true;
-				} else {
-					TEST(true); // Auto-throttling unspecified
-					if (autoThrottlingEnabled.get().present()) {
-						TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
-						    .detail("Value", autoThrottlingEnabled.get().get());
-					}
-					self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
-					if (!committed)
-						tr.set(tagThrottleAutoEnabledKey, LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
-				}
-
-				RkTagThrottleCollection updatedTagThrottles;
-
-				TraceEvent("RatekeeperReadThrottledTags", self->id)
-				    .detail("NumThrottledTags", throttledTagKeys.get().size());
-				for (auto entry : throttledTagKeys.get()) {
-					TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
-					TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
-
-					ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
-
-					if (tagValue.expirationTime == 0 || tagValue.expirationTime > now() + tagValue.initialDuration) {
-						TEST(true); // Converting tag throttle duration to absolute time
-						tagValue.expirationTime = now() + tagValue.initialDuration;
-						BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
-						wr << tagValue;
-						state Value value = wr.toValue();
-
-						tr.set(entry.key, value);
-					}
-
-					if (tagValue.expirationTime > now()) {
-						TransactionTag tag = *tagKey.tags.begin();
-						Optional<ClientTagThrottleLimits> oldLimits =
-						    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
-
-						if (tagKey.throttleType == TagThrottleType::AUTO) {
-							updatedTagThrottles.autoThrottleTag(
-							    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
-							if (tagValue.reason == TagThrottledReason::BUSY_READ) {
-								updatedTagThrottles.busyReadTagCount++;
-							} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
-								updatedTagThrottles.busyWriteTagCount++;
-							}
-						} else {
-							updatedTagThrottles.manualThrottleTag(
-							    self->id, tag, tagKey.priority, tagValue.tpsRate, tagValue.expirationTime, oldLimits);
-						}
-					}
-				}
-
-				self->throttledTags = std::move(updatedTagThrottles);
-				++self->throttledTagChangeId;
-
-				state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
-				wait(tr.commit());
-				committed = true;
-
-				wait(watchFuture);
-				TraceEvent("RatekeeperThrottleSignaled", self->id).log();
-				TEST(true); // Tag throttle changes detected
-				break;
-			} catch (Error& e) {
-				TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
-				wait(tr.onError(e));
-			}
-		}
-	}
-}
-
-Future<Void> refreshStorageServerCommitCost(RatekeeperData* self) {
-	if (self->lastBusiestCommitTagPick == 0) { // the first call should be skipped
-		self->lastBusiestCommitTagPick = now();
-		return Void();
-	}
-	double elapsed = now() - self->lastBusiestCommitTagPick;
-	// for each SS, select the busiest commit tag from ssTrTagCommitCost
-	for (auto it = self->storageQueueInfo.begin(); it != self->storageQueueInfo.end(); ++it) {
-		it->value.busiestWriteTag.reset();
-		TransactionTag busiestTag;
-		TransactionCommitCostEstimation maxCost;
-		double maxRate = 0, maxBusyness = 0;
-		for (const auto& [tag, cost] : it->value.tagCostEst) {
-			double rate = cost.getCostSum() / elapsed;
-			if (rate > maxRate) {
-				busiestTag = tag;
-				maxRate = rate;
-				maxCost = cost;
-			}
-		}
-		if (maxRate > SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE) {
-			it->value.busiestWriteTag = busiestTag;
-			// TraceEvent("RefreshSSCommitCost").detail("TotalWriteCost", it->value.totalWriteCost).detail("TotalWriteOps",it->value.totalWriteOps);
-			ASSERT(it->value.totalWriteCosts > 0);
-			maxBusyness = double(maxCost.getCostSum()) / it->value.totalWriteCosts;
-			it->value.busiestWriteTagFractionalBusyness = maxBusyness;
-			it->value.busiestWriteTagRate = maxRate;
-		}
-
-		TraceEvent("BusiestWriteTag", it->key)
-		    .detail("Elapsed", elapsed)
-		    .detail("Tag", printable(busiestTag))
-		    .detail("TagOps", maxCost.getOpsSum())
-		    .detail("TagCost", maxCost.getCostSum())
-		    .detail("TotalCost", it->value.totalWriteCosts)
-		    .detail("Reported", it->value.busiestWriteTag.present())
-		    .trackLatest(it->value.busiestWriteTagEventHolder->trackingKey);
-
-		// reset statistics
-		it->value.tagCostEst.clear();
-		it->value.totalWriteOps = 0;
-		it->value.totalWriteCosts = 0;
-	}
-	self->lastBusiestCommitTagPick = now();
-	return Void();
-}
-
-void tryAutoThrottleTag(RatekeeperData* self,
-                        TransactionTag tag,
-                        double rate,
-                        double busyness,
-                        TagThrottledReason reason) {
-	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
-	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
-	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
-		TEST(true); // Transaction tag auto-throttled
-		Optional<double> clientRate = self->throttledTags.autoThrottleTag(self->id, tag, busyness);
-		if (clientRate.present()) {
-			TagSet tags;
-			tags.addTag(tag);
-
-			Reference<DatabaseContext> db = Reference<DatabaseContext>::addRef(self->db.getPtr());
-			self->addActor.send(ThrottleApi::throttleTags(db,
-			                                              tags,
-			                                              clientRate.get(),
-			                                              SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                              TagThrottleType::AUTO,
-			                                              TransactionPriority::DEFAULT,
-			                                              now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                              reason));
-		}
-	}
-}
-
-void tryAutoThrottleTag(RatekeeperData* self,
-                        StorageQueueInfo& ss,
-                        int64_t storageQueue,
-                        int64_t storageDurabilityLag) {
-	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
-	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
-	// future
-	if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
-	    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
-		if (ss.busiestWriteTag.present()) {
-			tryAutoThrottleTag(self,
-			                   ss.busiestWriteTag.get(),
-			                   ss.busiestWriteTagRate,
-			                   ss.busiestWriteTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_WRITE);
-		}
-		if (ss.busiestReadTag.present()) {
-			tryAutoThrottleTag(self,
-			                   ss.busiestReadTag.get(),
-			                   ss.busiestReadTagRate,
-			                   ss.busiestReadTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_READ);
-		}
-	}
-}
-
-void updateRate(RatekeeperData* self, RatekeeperLimits* limits) {
-	// double controlFactor = ;  // dt / eFoldingTime
-
-	double actualTps = self->smoothReleasedTransactions.smoothRate();
-	self->actualTpsMetric = (int64_t)actualTps;
-	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this
-	// value
-	actualTps = std::max(std::max(1.0, actualTps),
-	                     self->smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
-
-	if (self->actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
-		self->actualTpsHistory.pop_front();
-	}
-	self->actualTpsHistory.push_back(actualTps);
-
-	limits->tpsLimit = std::numeric_limits<double>::infinity();
-	UID reasonID = UID();
-	limitReason_t limitReason = limitReason_t::unlimited;
-
-	int sscount = 0;
-
-	int64_t worstFreeSpaceStorageServer = std::numeric_limits<int64_t>::max();
-	int64_t worstStorageQueueStorageServer = 0;
-	int64_t limitingStorageQueueStorageServer = 0;
-	int64_t worstDurabilityLag = 0;
-
-	std::multimap<double, StorageQueueInfo*> storageTpsLimitReverseIndex;
-	std::multimap<int64_t, StorageQueueInfo*> storageDurabilityLagReverseIndex;
-
-	std::map<UID, limitReason_t> ssReasons;
-
-	bool printRateKeepLimitReasonDetails =
-	    SERVER_KNOBS->RATEKEEPER_PRINT_LIMIT_REASON &&
-	    (deterministicRandom()->random01() < SERVER_KNOBS->RATEKEEPER_LIMIT_REASON_SAMPLE_RATE);
-
-	// Look at each storage server's write queue and local rate, compute and store the desired rate ratio
-	for (auto i = self->storageQueueInfo.begin(); i != self->storageQueueInfo.end(); ++i) {
-		auto& ss = i->value;
-		if (!ss.valid || (self->remoteDC.present() && ss.locality.dcId() == self->remoteDC))
-			continue;
-		++sscount;
-
-		limitReason_t ssLimitReason = limitReason_t::unlimited;
-
-		int64_t minFreeSpace =
-		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
-		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * ss.smoothTotalSpace.smoothTotal()));
-
-		worstFreeSpaceStorageServer =
-		    std::min(worstFreeSpaceStorageServer, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace);
-
-		int64_t springBytes = std::max<int64_t>(
-		    1, std::min<int64_t>(limits->storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
-		int64_t targetBytes = std::max<int64_t>(
-		    1, std::min(limits->storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
-		if (targetBytes != limits->storageTargetBytes) {
-			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
-				ssLimitReason = limitReason_t::storage_server_min_free_space;
-			} else {
-				ssLimitReason = limitReason_t::storage_server_min_free_space_ratio;
-			}
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("Reason", ssLimitReason)
-				    .detail("SSID", ss.id)
-				    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
-				    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
-				    .detail("TargetBytes", targetBytes)
-				    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
-				    .detail("MinFreeSpace", minFreeSpace);
-			}
-		}
-
-		int64_t storageQueue = ss.lastReply.bytesInput - ss.smoothDurableBytes.smoothTotal();
-		worstStorageQueueStorageServer = std::max(worstStorageQueueStorageServer, storageQueue);
-
-		int64_t storageDurabilityLag = ss.smoothLatestVersion.smoothTotal() - ss.smoothDurableVersion.smoothTotal();
-		worstDurabilityLag = std::max(worstDurabilityLag, storageDurabilityLag);
-
-		storageDurabilityLagReverseIndex.insert(std::make_pair(-1 * storageDurabilityLag, &ss));
-
-		auto& ssMetrics = self->healthMetrics.storageStats[ss.id];
-		ssMetrics.storageQueue = storageQueue;
-		ssMetrics.storageDurabilityLag = storageDurabilityLag;
-		ssMetrics.cpuUsage = ss.lastReply.cpuUsage;
-		ssMetrics.diskUsage = ss.lastReply.diskUsage;
-
-		double targetRateRatio = std::min((storageQueue - targetBytes + springBytes) / (double)springBytes, 2.0);
-
-		if (limits->priority == TransactionPriority::DEFAULT) {
-			tryAutoThrottleTag(self, ss, storageQueue, storageDurabilityLag);
-		}
-
-		double inputRate = ss.smoothInputBytes.smoothRate();
-		// inputRate = std::max( inputRate, actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE );
-
-		/*if( deterministicRandom()->random01() < 0.1 ) {
-		    std::string name = "RatekeeperUpdateRate" + limits.context;
-		    TraceEvent(name, ss.id)
-		        .detail("MinFreeSpace", minFreeSpace)
-		        .detail("SpringBytes", springBytes)
-		        .detail("TargetBytes", targetBytes)
-		        .detail("SmoothTotalSpaceTotal", ss.smoothTotalSpace.smoothTotal())
-		        .detail("SmoothFreeSpaceTotal", ss.smoothFreeSpace.smoothTotal())
-		        .detail("LastReplyBytesInput", ss.lastReply.bytesInput)
-		        .detail("SmoothDurableBytesTotal", ss.smoothDurableBytes.smoothTotal())
-		        .detail("TargetRateRatio", targetRateRatio)
-		        .detail("SmoothInputBytesRate", ss.smoothInputBytes.smoothRate())
-		        .detail("ActualTPS", actualTps)
-		        .detail("InputRate", inputRate)
-		        .detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate())
-		        .detail("B", b);
-		}*/
-
-		// Don't let any storage server use up its target bytes faster than its MVCC window!
-		double maxBytesPerSecond =
-		    (targetBytes - springBytes) /
-		    ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0);
-		double limitTps = std::min(actualTps * maxBytesPerSecond / std::max(1.0e-8, inputRate),
-		                           maxBytesPerSecond * SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-		if (ssLimitReason == limitReason_t::unlimited)
-			ssLimitReason = limitReason_t::storage_server_write_bandwidth_mvcc;
-
-		if (targetRateRatio > 0 && inputRate > 0) {
-			ASSERT(inputRate != 0);
-			double smoothedRate =
-			    std::max(ss.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-			double x = smoothedRate / (inputRate * targetRateRatio);
-			double lim = actualTps * x;
-			if (lim < limitTps) {
-				double oldLimitTps = limitTps;
-				limitTps = lim;
-				if (ssLimitReason == limitReason_t::unlimited ||
-				    ssLimitReason == limitReason_t::storage_server_write_bandwidth_mvcc) {
-					if (printRateKeepLimitReasonDetails) {
-						TraceEvent("RatekeeperLimitReasonDetails")
-						    .detail("Reason", limitReason_t::storage_server_write_queue_size)
-						    .detail("FromReason", ssLimitReason)
-						    .detail("SSID", ss.id)
-						    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
-						    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
-						    .detail("LimitsStorageSpringBytes", limits->storageSpringBytes)
-						    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
-						    .detail("MinFreeSpace", minFreeSpace)
-						    .detail("SSLastReplyBytesInput", ss.lastReply.bytesInput)
-						    .detail("SSSmoothDurableBytes", ss.smoothDurableBytes.smoothTotal())
-						    .detail("StorageQueue", storageQueue)
-						    .detail("TargetBytes", targetBytes)
-						    .detail("SpringBytes", springBytes)
-						    .detail("SSVerySmoothDurableBytesSmoothRate", ss.verySmoothDurableBytes.smoothRate())
-						    .detail("SmoothedRate", smoothedRate)
-						    .detail("X", x)
-						    .detail("ActualTps", actualTps)
-						    .detail("Lim", lim)
-						    .detail("LimitTps", oldLimitTps)
-						    .detail("InputRate", inputRate)
-						    .detail("TargetRateRatio", targetRateRatio);
-					}
-					ssLimitReason = limitReason_t::storage_server_write_queue_size;
-				}
-			}
-		}
-
-		storageTpsLimitReverseIndex.insert(std::make_pair(limitTps, &ss));
-
-		if (limitTps < limits->tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space ||
-		                                    ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
-			reasonID = ss.id;
-			limits->tpsLimit = limitTps;
-			limitReason = ssLimitReason;
-		}
-
-		ssReasons[ss.id] = ssLimitReason;
-	}
-
-	std::set<Optional<Standalone<StringRef>>> ignoredMachines;
-	for (auto ss = storageTpsLimitReverseIndex.begin();
-	     ss != storageTpsLimitReverseIndex.end() && ss->first < limits->tpsLimit;
-	     ++ss) {
-		if (ignoredMachines.size() <
-		    std::min(self->configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
-			ignoredMachines.insert(ss->second->locality.zoneId());
-			continue;
-		}
-		if (ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
-			continue;
-		}
-
-		limitingStorageQueueStorageServer =
-		    ss->second->lastReply.bytesInput - ss->second->smoothDurableBytes.smoothTotal();
-		limits->tpsLimit = ss->first;
-		reasonID = storageTpsLimitReverseIndex.begin()->second->id; // Although we aren't controlling based on the worst
-		                                                            // SS, we still report it as the limiting process
-		limitReason = ssReasons[reasonID];
-		break;
-	}
-
-	// Calculate limited durability lag
-	int64_t limitingDurabilityLag = 0;
-
-	std::set<Optional<Standalone<StringRef>>> ignoredDurabilityLagMachines;
-	for (auto ss = storageDurabilityLagReverseIndex.begin(); ss != storageDurabilityLagReverseIndex.end(); ++ss) {
-		if (ignoredDurabilityLagMachines.size() <
-		    std::min(self->configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
-			ignoredDurabilityLagMachines.insert(ss->second->locality.zoneId());
-			continue;
-		}
-		if (ignoredDurabilityLagMachines.count(ss->second->locality.zoneId()) > 0) {
-			continue;
-		}
-
-		limitingDurabilityLag = -1 * ss->first;
-		if (limitingDurabilityLag > limits->durabilityLagTargetVersions &&
-		    self->actualTpsHistory.size() > SERVER_KNOBS->NEEDED_TPS_HISTORY_SAMPLES) {
-			if (limits->durabilityLagLimit == std::numeric_limits<double>::infinity()) {
-				double maxTps = 0;
-				for (int i = 0; i < self->actualTpsHistory.size(); i++) {
-					maxTps = std::max(maxTps, self->actualTpsHistory[i]);
-				}
-				limits->durabilityLagLimit = SERVER_KNOBS->INITIAL_DURABILITY_LAG_MULTIPLIER * maxTps;
-			}
-			if (limitingDurabilityLag > limits->lastDurabilityLag) {
-				limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_REDUCTION_RATE * limits->durabilityLagLimit;
-			}
-			if (limits->durabilityLagLimit < limits->tpsLimit) {
-				if (printRateKeepLimitReasonDetails) {
-					TraceEvent("RatekeeperLimitReasonDetails")
-					    .detail("SSID", ss->second->id)
-					    .detail("Reason", limitReason_t::storage_server_durability_lag)
-					    .detail("LimitsDurabilityLagLimit", limits->durabilityLagLimit)
-					    .detail("LimitsTpsLimit", limits->tpsLimit)
-					    .detail("LimitingDurabilityLag", limitingDurabilityLag)
-					    .detail("LimitsLastDurabilityLag", limits->lastDurabilityLag);
-				}
-				limits->tpsLimit = limits->durabilityLagLimit;
-				limitReason = limitReason_t::storage_server_durability_lag;
-			}
-		} else if (limits->durabilityLagLimit != std::numeric_limits<double>::infinity() &&
-		           limitingDurabilityLag >
-		               limits->durabilityLagTargetVersions - SERVER_KNOBS->DURABILITY_LAG_UNLIMITED_THRESHOLD) {
-			limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_INCREASE_RATE * limits->durabilityLagLimit;
-		} else {
-			limits->durabilityLagLimit = std::numeric_limits<double>::infinity();
-		}
-		limits->lastDurabilityLag = limitingDurabilityLag;
-		break;
-	}
-
-	self->healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
-	self->healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
-	self->healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
-	self->healthMetrics.limitingStorageDurabilityLag = limitingDurabilityLag;
-
-	double writeToReadLatencyLimit = 0;
-	Version worstVersionLag = 0;
-	Version limitingVersionLag = 0;
-
-	{
-		Version minSSVer = std::numeric_limits<Version>::max();
-		Version minLimitingSSVer = std::numeric_limits<Version>::max();
-		for (const auto& it : self->storageQueueInfo) {
-			auto& ss = it.value;
-			if (!ss.valid || (self->remoteDC.present() && ss.locality.dcId() == self->remoteDC))
-				continue;
-
-			minSSVer = std::min(minSSVer, ss.lastReply.version);
-
-			// Machines that ratekeeper isn't controlling can fall arbitrarily far behind
-			if (ignoredMachines.count(it.value.locality.zoneId()) == 0) {
-				minLimitingSSVer = std::min(minLimitingSSVer, ss.lastReply.version);
-			}
-		}
-
-		Version maxTLVer = std::numeric_limits<Version>::min();
-		for (const auto& it : self->tlogQueueInfo) {
-			auto& tl = it.value;
-			if (!tl.valid)
-				continue;
-			maxTLVer = std::max(maxTLVer, tl.lastReply.v);
-		}
-
-		if (minSSVer != std::numeric_limits<Version>::max() && maxTLVer != std::numeric_limits<Version>::min()) {
-			// writeToReadLatencyLimit: 0 = infinte speed; 1 = TL durable speed ; 2 = half TL durable speed
-			writeToReadLatencyLimit =
-			    ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference / 2) / (limits->maxVersionDifference / 4);
-			worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
-			limitingVersionLag = std::max((Version)0, maxTLVer - minLimitingSSVer);
-		}
-	}
-
-	int64_t worstFreeSpaceTLog = std::numeric_limits<int64_t>::max();
-	int64_t worstStorageQueueTLog = 0;
-	int tlcount = 0;
-	for (auto& it : self->tlogQueueInfo) {
-		auto& tl = it.value;
-		if (!tl.valid)
-			continue;
-		++tlcount;
-
-		limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue;
-
-		int64_t minFreeSpace =
-		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
-		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.smoothTotalSpace.smoothTotal()));
-
-		worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace);
-
-		int64_t springBytes = std::max<int64_t>(
-		    1, std::min<int64_t>(limits->logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
-		int64_t targetBytes = std::max<int64_t>(
-		    1, std::min(limits->logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
-		if (targetBytes != limits->logTargetBytes) {
-			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
-				tlogLimitReason = limitReason_t::log_server_min_free_space;
-			} else {
-				tlogLimitReason = limitReason_t::log_server_min_free_space_ratio;
-			}
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("TLogID", tl.id)
-				    .detail("Reason", tlogLimitReason)
-				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-				    .detail("TargetBytes", targetBytes)
-				    .detail("MinFreeSpace", minFreeSpace);
-			}
-		}
-
-		int64_t queue = tl.lastReply.bytesInput - tl.smoothDurableBytes.smoothTotal();
-		self->healthMetrics.tLogQueue[tl.id] = queue;
-		int64_t b = queue - targetBytes;
-		worstStorageQueueTLog = std::max(worstStorageQueueTLog, queue);
-
-		if (tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2) {
-			if (now() - self->lastWarning > 5.0) {
-				self->lastWarning = now();
-				TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero", self->id).detail("ReasonId", tl.id);
-			}
-			reasonID = tl.id;
-			limitReason = limitReason_t::log_server_min_free_space;
-			limits->tpsLimit = 0.0;
-		}
-
-		double targetRateRatio = std::min((b + springBytes) / (double)springBytes, 2.0);
-
-		if (writeToReadLatencyLimit > targetRateRatio) {
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("TLogID", tl.id)
-				    .detail("Reason", limitReason_t::storage_server_readable_behind)
-				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-				    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
-				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-				    .detail("SpringBytes", springBytes)
-				    .detail("TargetBytes", targetBytes)
-				    .detail("TLLastReplyBytesInput", tl.lastReply.bytesInput)
-				    .detail("TLSmoothDurableBytes", tl.smoothDurableBytes.smoothTotal())
-				    .detail("Queue", queue)
-				    .detail("B", b)
-				    .detail("TargetRateRatio", targetRateRatio)
-				    .detail("WriteToReadLatencyLimit", writeToReadLatencyLimit)
-				    .detail("MinFreeSpace", minFreeSpace)
-				    .detail("LimitsMaxVersionDifference", limits->maxVersionDifference);
-			}
-			targetRateRatio = writeToReadLatencyLimit;
-			tlogLimitReason = limitReason_t::storage_server_readable_behind;
-		}
-
-		double inputRate = tl.smoothInputBytes.smoothRate();
-
-		if (targetRateRatio > 0) {
-			double smoothedRate =
-			    std::max(tl.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-			double x = smoothedRate / (inputRate * targetRateRatio);
-			if (targetRateRatio < .75) //< FIXME: KNOB for 2.0
-				x = std::max(x, 0.95);
-			double lim = actualTps * x;
-			if (lim < limits->tpsLimit) {
-				limits->tpsLimit = lim;
-				reasonID = tl.id;
-				limitReason = tlogLimitReason;
-			}
-		}
-		if (inputRate > 0) {
-			// Don't let any tlogs use up its target bytes faster than its MVCC window!
-			double x =
-			    ((targetBytes - springBytes) /
-			     ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-			      2.0)) /
-			    inputRate;
-			double lim = actualTps * x;
-			if (lim < limits->tpsLimit) {
-				if (printRateKeepLimitReasonDetails) {
-					TraceEvent("RatekeeperLimitReasonDetails")
-					    .detail("Reason", limitReason_t::log_server_mvcc_write_bandwidth)
-					    .detail("TLogID", tl.id)
-					    .detail("MinFreeSpace", minFreeSpace)
-					    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-					    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-					    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
-					    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-					    .detail("SpringBytes", springBytes)
-					    .detail("TargetBytes", targetBytes)
-					    .detail("InputRate", inputRate)
-					    .detail("X", x)
-					    .detail("ActualTps", actualTps)
-					    .detail("Lim", lim)
-					    .detail("LimitsTpsLimit", limits->tpsLimit);
-				}
-				limits->tpsLimit = lim;
-				reasonID = tl.id;
-				limitReason = limitReason_t::log_server_mvcc_write_bandwidth;
-			}
-		}
-	}
-
-	self->healthMetrics.worstTLogQueue = worstStorageQueueTLog;
-
-	limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
-
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
-		limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
-	}
-
-	int64_t totalDiskUsageBytes = 0;
-	for (auto& t : self->tlogQueueInfo) {
-		if (t.value.valid) {
-			totalDiskUsageBytes += t.value.lastReply.storageBytes.used;
-		}
-	}
-	for (auto& s : self->storageQueueInfo) {
-		if (s.value.valid) {
-			totalDiskUsageBytes += s.value.lastReply.storageBytes.used;
-		}
-	}
-
-	if (now() - self->lastSSListFetchedTimestamp > SERVER_KNOBS->STORAGE_SERVER_LIST_FETCH_TIMEOUT) {
-		limits->tpsLimit = 0.0;
-		limitReason = limitReason_t::storage_server_list_fetch_failed;
-		reasonID = UID();
-		TraceEvent(SevWarnAlways, "RkSSListFetchTimeout", self->id).suppressFor(1.0);
-	} else if (limits->tpsLimit == std::numeric_limits<double>::infinity()) {
-		limits->tpsLimit = SERVER_KNOBS->RATEKEEPER_DEFAULT_LIMIT;
-	}
-
-	limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
-	limits->reasonMetric = limitReason;
-
-	if (deterministicRandom()->random01() < 0.1) {
-		const std::string& name = limits->rkUpdateEventCacheHolder.getPtr()->trackingKey;
-		TraceEvent(name.c_str(), self->id)
-		    .detail("TPSLimit", limits->tpsLimit)
-		    .detail("Reason", limitReason)
-		    .detail("ReasonServerID", reasonID == UID() ? std::string() : Traceable<UID>::toString(reasonID))
-		    .detail("ReleasedTPS", self->smoothReleasedTransactions.smoothRate())
-		    .detail("ReleasedBatchTPS", self->smoothBatchReleasedTransactions.smoothRate())
-		    .detail("TPSBasis", actualTps)
-		    .detail("StorageServers", sscount)
-		    .detail("GrvProxies", self->grvProxyInfo.size())
-		    .detail("TLogs", tlcount)
-		    .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer)
-		    .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog)
-		    .detail("WorstStorageServerQueue", worstStorageQueueStorageServer)
-		    .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer)
-		    .detail("WorstTLogQueue", worstStorageQueueTLog)
-		    .detail("TotalDiskUsageBytes", totalDiskUsageBytes)
-		    .detail("WorstStorageServerVersionLag", worstVersionLag)
-		    .detail("LimitingStorageServerVersionLag", limitingVersionLag)
-		    .detail("WorstStorageServerDurabilityLag", worstDurabilityLag)
-		    .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag)
-		    .detail("TagsAutoThrottled", self->throttledTags.autoThrottleCount())
-		    .detail("TagsAutoThrottledBusyRead", self->throttledTags.busyReadTagCount)
-		    .detail("TagsAutoThrottledBusyWrite", self->throttledTags.busyWriteTagCount)
-		    .detail("TagsManuallyThrottled", self->throttledTags.manualThrottleCount())
-		    .detail("AutoThrottlingEnabled", self->autoThrottlingEnabled)
-		    .trackLatest(name);
-	}
-}
-
-static void updateCommitCostEstimation(RatekeeperData* self,
-                                       UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation) {
-	for (auto it = self->storageQueueInfo.begin(); it != self->storageQueueInfo.end(); ++it) {
-		auto tagCostIt = costEstimation.find(it->key);
-		if (tagCostIt == costEstimation.end())
-			continue;
-		for (const auto& [tagName, cost] : tagCostIt->second) {
-			it->value.tagCostEst[tagName] += cost;
-			it->value.totalWriteCosts += cost.getCostSum();
-			it->value.totalWriteOps += cost.getOpsSum();
-		}
-	}
-}
-
-ACTOR Future<Void> configurationMonitor(RatekeeperData* self) {
-	loop {
-		state ReadYourWritesTransaction tr(self->db);
-
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
-				ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
-
-				self->configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
-
-				state Future<Void> watchFuture =
-				    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
-				    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
-				    tr.watch(failedLocalityVersionKey);
-				wait(tr.commit());
-				wait(watchFuture);
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-}
-
-ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	state RatekeeperData self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
-	state Future<Void> timeout = Void();
-	state std::vector<Future<Void>> tlogTrackers;
-	state std::vector<TLogInterface> tlogInterfs;
-	state Promise<Void> err;
-	state Future<Void> collection = actorCollection(self.addActor.getFuture());
-
-	TraceEvent("RatekeeperStarting", rkInterf.id());
-	self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
-	self.addActor.send(configurationMonitor(&self));
-
-	PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
-	self.addActor.send(monitorServerListChange(&self, serverChanges));
-	self.addActor.send(trackEachStorageServer(&self, serverChanges.getFuture()));
-	self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
-
-	self.addActor.send(monitorThrottlingChanges(&self));
-	RatekeeperData* selfPtr = &self; // let flow compiler capture self
-	self.addActor.send(
-	    recurring([selfPtr]() { refreshStorageServerCommitCost(selfPtr); }, SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
-
-	TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
-	    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
-	    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
-	    .detail("Rate",
-	            (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
-	                ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-	                 2.0));
-
-	TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
-	    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
-	    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
-	    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
-	    .detail("Rate",
-	            (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
-	                ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-	                 2.0));
-
-	tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-	tlogTrackers.reserve(tlogInterfs.size());
-	for (int i = 0; i < tlogInterfs.size(); i++) {
-		tlogTrackers.push_back(splitError(trackTLogQueueInfo(&self, tlogInterfs[i]), err));
-	}
-
-	self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
-
-	try {
-		state bool lastLimited = false;
-		loop choose {
-			when(wait(timeout)) {
-				updateRate(&self, &self.normalLimits);
-				updateRate(&self, &self.batchLimits);
-
-				lastLimited = self.smoothReleasedTransactions.smoothRate() >
-				              SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
-				double tooOld = now() - 1.0;
-				for (auto p = self.grvProxyInfo.begin(); p != self.grvProxyInfo.end();) {
-					if (p->second.lastUpdateTime < tooOld)
-						p = self.grvProxyInfo.erase(p);
-					else
-						++p;
-				}
-				timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
-			}
-			when(GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
-				GetRateInfoReply reply;
-
-				auto& p = self.grvProxyInfo[req.requesterID];
-				//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
-				if (p.totalTransactions > 0) {
-					self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
-
-					for (auto tag : req.throttledTagCounts) {
-						self.throttledTags.addRequests(tag.first, tag.second);
-					}
-				}
-				if (p.batchTransactions > 0) {
-					self.smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions - p.batchTransactions);
-				}
-
-				p.totalTransactions = req.totalReleasedTransactions;
-				p.batchTransactions = req.batchReleasedTransactions;
-				p.lastUpdateTime = now();
-
-				reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
-				reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
-				reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
-
-				if (p.lastThrottledTagChangeId != self.throttledTagChangeId ||
-				    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
-					p.lastThrottledTagChangeId = self.throttledTagChangeId;
-					p.lastTagPushTime = now();
-
-					reply.throttledTags = self.throttledTags.getClientRates(self.autoThrottlingEnabled);
-					bool returningTagsToProxy = reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
-					TEST(returningTagsToProxy); // Returning tag throttles to a proxy
-				}
-
-				reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
-				reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
-				reply.healthMetrics.batchLimited = lastLimited;
-
-				req.reply.send(reply);
-			}
-			when(HaltRatekeeperRequest req = waitNext(rkInterf.haltRatekeeper.getFuture())) {
-				req.reply.send(Void());
-				TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
-				break;
-			}
-			when(ReportCommitCostEstimationRequest req = waitNext(rkInterf.reportCommitCostEstimation.getFuture())) {
-				updateCommitCostEstimation(&self, req.ssTrTagCommitCost);
-				req.reply.send(Void());
-			}
-			when(wait(err.getFuture())) {}
-			when(wait(dbInfo->onChange())) {
-				if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
-					tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-					tlogTrackers = std::vector<Future<Void>>();
-					for (int i = 0; i < tlogInterfs.size(); i++)
-						tlogTrackers.push_back(splitError(trackTLogQueueInfo(&self, tlogInterfs[i]), err));
-				}
-				self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
-			}
-			when(wait(collection)) {
-				ASSERT(false);
-				throw internal_error();
-			}
-		}
-	} catch (Error& err) {
-		TraceEvent("RatekeeperDied", rkInterf.id()).error(err, true);
-	}
-	return Void();
-}
diff --git a/fdbserver/RatekeeperData.actor.cpp b/fdbserver/RatekeeperData.actor.cpp
index 09e4c56f1f..e644240a16 100644
--- a/fdbserver/RatekeeperData.actor.cpp
+++ b/fdbserver/RatekeeperData.actor.cpp
@@ -21,6 +21,39 @@
 #include "fdbserver/RatekeeperData.h"
 #include "flow/actorcompiler.h" // must be last include
 
+const char* limitReasonName[] = { "workload",
+	                              "storage_server_write_queue_size",
+	                              "storage_server_write_bandwidth_mvcc",
+	                              "storage_server_readable_behind",
+	                              "log_server_mvcc_write_bandwidth",
+	                              "log_server_write_queue",
+	                              "storage_server_min_free_space",
+	                              "storage_server_min_free_space_ratio",
+	                              "log_server_min_free_space",
+	                              "log_server_min_free_space_ratio",
+	                              "storage_server_durability_lag",
+	                              "storage_server_list_fetch_failed" };
+static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
+
+int limitReasonEnd = limitReason_t_end;
+
+// NOTE: This has a corresponding table in Script.cs (see RatekeeperReason graph)
+// IF UPDATING THIS ARRAY, UPDATE SCRIPT.CS!
+const char* limitReasonDesc[] = { "Workload or read performance.",
+	                              "Storage server performance (storage queue).",
+	                              "Storage server MVCC memory.",
+	                              "Storage server version falling behind.",
+	                              "Log server MVCC memory.",
+	                              "Storage server performance (log queue).",
+	                              "Storage server running out of space (approaching 100MB limit).",
+	                              "Storage server running out of space (approaching 5% limit).",
+	                              "Log server running out of space (approaching 100MB limit).",
+	                              "Log server running out of space (approaching 5% limit).",
+	                              "Storage server durable version falling behind.",
+	                              "Unable to fetch storage server list." };
+
+static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
+
 ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
 	try {
 		wait(in);
@@ -228,6 +261,258 @@ public:
 		}
 	}
 
+	ACTOR static Future<Void> monitorThrottlingChanges(RatekeeperData* self) {
+		state bool committed = false;
+		loop {
+			state ReadYourWritesTransaction tr(self->db);
+
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
+					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
+
+					if (!committed) {
+						BinaryWriter limitWriter(Unversioned());
+						limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
+						tr.set(tagThrottleLimitKey, limitWriter.toValue());
+					}
+
+					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
+
+					if (autoThrottlingEnabled.get().present() &&
+					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
+						TEST(true); // Auto-throttling disabled
+						if (self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
+						}
+						self->autoThrottlingEnabled = false;
+					} else if (autoThrottlingEnabled.get().present() &&
+					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
+						TEST(true); // Auto-throttling enabled
+						if (!self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
+						}
+						self->autoThrottlingEnabled = true;
+					} else {
+						TEST(true); // Auto-throttling unspecified
+						if (autoThrottlingEnabled.get().present()) {
+							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
+							    .detail("Value", autoThrottlingEnabled.get().get());
+						}
+						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
+						if (!committed)
+							tr.set(tagThrottleAutoEnabledKey,
+							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
+					}
+
+					RkTagThrottleCollection updatedTagThrottles;
+
+					TraceEvent("RatekeeperReadThrottledTags", self->id)
+					    .detail("NumThrottledTags", throttledTagKeys.get().size());
+					for (auto entry : throttledTagKeys.get()) {
+						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
+						TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
+
+						ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
+
+						if (tagValue.expirationTime == 0 ||
+						    tagValue.expirationTime > now() + tagValue.initialDuration) {
+							TEST(true); // Converting tag throttle duration to absolute time
+							tagValue.expirationTime = now() + tagValue.initialDuration;
+							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
+							wr << tagValue;
+							state Value value = wr.toValue();
+
+							tr.set(entry.key, value);
+						}
+
+						if (tagValue.expirationTime > now()) {
+							TransactionTag tag = *tagKey.tags.begin();
+							Optional<ClientTagThrottleLimits> oldLimits =
+							    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
+
+							if (tagKey.throttleType == TagThrottleType::AUTO) {
+								updatedTagThrottles.autoThrottleTag(
+								    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
+								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
+									updatedTagThrottles.busyReadTagCount++;
+								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
+									updatedTagThrottles.busyWriteTagCount++;
+								}
+							} else {
+								updatedTagThrottles.manualThrottleTag(self->id,
+								                                      tag,
+								                                      tagKey.priority,
+								                                      tagValue.tpsRate,
+								                                      tagValue.expirationTime,
+								                                      oldLimits);
+							}
+						}
+					}
+
+					self->throttledTags = std::move(updatedTagThrottles);
+					++self->throttledTagChangeId;
+
+					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
+					wait(tr.commit());
+					committed = true;
+
+					wait(watchFuture);
+					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
+					TEST(true); // Tag throttle changes detected
+					break;
+				} catch (Error& e) {
+					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+		state RatekeeperData self(rkInterf.id(),
+		                          openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
+		state Future<Void> timeout = Void();
+		state std::vector<Future<Void>> tlogTrackers;
+		state std::vector<TLogInterface> tlogInterfs;
+		state Promise<Void> err;
+		state Future<Void> collection = actorCollection(self.addActor.getFuture());
+
+		TraceEvent("RatekeeperStarting", rkInterf.id());
+		self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
+		self.addActor.send(self.configurationMonitor());
+
+		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
+		self.addActor.send(self.monitorServerListChange(serverChanges));
+		self.addActor.send(self.trackEachStorageServer(serverChanges.getFuture()));
+		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
+
+		self.addActor.send(self.monitorThrottlingChanges());
+		RatekeeperData* selfPtr = &self; // let flow compiler capture self
+		self.addActor.send(recurring([selfPtr]() { selfPtr->refreshStorageServerCommitCost(); },
+		                             SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
+
+		TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
+		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
+		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
+		    .detail(
+		        "Rate",
+		        (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
+		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+		             2.0));
+
+		TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
+		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
+		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
+		    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
+		    .detail(
+		        "Rate",
+		        (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
+		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+		             2.0));
+
+		tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+		tlogTrackers.reserve(tlogInterfs.size());
+		for (int i = 0; i < tlogInterfs.size(); i++) {
+			tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
+		}
+
+		self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+
+		try {
+			state bool lastLimited = false;
+			loop choose {
+				when(wait(timeout)) {
+					self.updateRate(&self.normalLimits);
+					self.updateRate(&self.batchLimits);
+
+					lastLimited = self.smoothReleasedTransactions.smoothRate() >
+					              SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
+					double tooOld = now() - 1.0;
+					for (auto p = self.grvProxyInfo.begin(); p != self.grvProxyInfo.end();) {
+						if (p->second.lastUpdateTime < tooOld)
+							p = self.grvProxyInfo.erase(p);
+						else
+							++p;
+					}
+					timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
+				}
+				when(GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
+					GetRateInfoReply reply;
+
+					auto& p = self.grvProxyInfo[req.requesterID];
+					//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
+					if (p.totalTransactions > 0) {
+						self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
+
+						for (auto tag : req.throttledTagCounts) {
+							self.throttledTags.addRequests(tag.first, tag.second);
+						}
+					}
+					if (p.batchTransactions > 0) {
+						self.smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions -
+						                                              p.batchTransactions);
+					}
+
+					p.totalTransactions = req.totalReleasedTransactions;
+					p.batchTransactions = req.batchReleasedTransactions;
+					p.lastUpdateTime = now();
+
+					reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
+					reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
+					reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
+
+					if (p.lastThrottledTagChangeId != self.throttledTagChangeId ||
+					    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
+						p.lastThrottledTagChangeId = self.throttledTagChangeId;
+						p.lastTagPushTime = now();
+
+						reply.throttledTags = self.throttledTags.getClientRates(self.autoThrottlingEnabled);
+						bool returningTagsToProxy =
+						    reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
+						TEST(returningTagsToProxy); // Returning tag throttles to a proxy
+					}
+
+					reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
+					reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
+					reply.healthMetrics.batchLimited = lastLimited;
+
+					req.reply.send(reply);
+				}
+				when(HaltRatekeeperRequest req = waitNext(rkInterf.haltRatekeeper.getFuture())) {
+					req.reply.send(Void());
+					TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
+					break;
+				}
+				when(ReportCommitCostEstimationRequest req =
+				         waitNext(rkInterf.reportCommitCostEstimation.getFuture())) {
+					self.updateCommitCostEstimation(req.ssTrTagCommitCost);
+					req.reply.send(Void());
+				}
+				when(wait(err.getFuture())) {}
+				when(wait(dbInfo->onChange())) {
+					if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
+						tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+						tlogTrackers = std::vector<Future<Void>>();
+						for (int i = 0; i < tlogInterfs.size(); i++)
+							tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
+					}
+					self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+				}
+				when(wait(collection)) {
+					ASSERT(false);
+					throw internal_error();
+				}
+			}
+		} catch (Error& err) {
+			TraceEvent("RatekeeperDied", rkInterf.id()).error(err, true);
+		}
+		return Void();
+	}
+
 }; // class RatekeeperDataImpl
 
 Future<Void> RatekeeperData::configurationMonitor() {
@@ -252,6 +537,14 @@ Future<Void> RatekeeperData::trackTLogQueueInfo(TLogInterface tli) {
 	return RatekeeperDataImpl::trackTLogQueueInfo(this, tli);
 }
 
+Future<Void> RatekeeperData::monitorThrottlingChanges() {
+	return RatekeeperDataImpl::monitorThrottlingChanges(this);
+}
+
+Future<Void> RatekeeperData::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	return RatekeeperDataImpl::run(rkInterf, dbInfo);
+}
+
 RatekeeperData::RatekeeperData(UID id, Database db)
   : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
@@ -863,3 +1156,8 @@ void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQue
 		}
 	}
 }
+
+ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	wait(RatekeeperData::run(rkInterf, dbInfo));
+	return Void();
+}
diff --git a/fdbserver/RatekeeperData.h b/fdbserver/RatekeeperData.h
index 1247e03a0e..4340796a2e 100644
--- a/fdbserver/RatekeeperData.h
+++ b/fdbserver/RatekeeperData.h
@@ -563,6 +563,7 @@ struct RatekeeperData {
 
 	void tryAutoThrottleTag(TransactionTag, double rate, double busyness, TagThrottledReason);
 	void tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
+	Future<Void> monitorThrottlingChanges();
 
 	static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 };

From d4b4479399dd3981adc1640c98702aac146c0298 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 12:35:50 -0800
Subject: [PATCH 061/210] Rename RatekeeperData.actor.cpp to
 Ratekeeper.actor.cpp

---
 fdbserver/CMakeLists.txt                     |    3 +-
 fdbserver/Ratekeeper.actor.cpp               | 1162 ++++++++++++++++-
 fdbserver/{RatekeeperData.h => Ratekeeper.h} |    0
 fdbserver/RatekeeperData.actor.cpp           | 1163 ------------------
 4 files changed, 1146 insertions(+), 1182 deletions(-)
 rename fdbserver/{RatekeeperData.h => Ratekeeper.h} (100%)
 delete mode 100644 fdbserver/RatekeeperData.actor.cpp

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 38d3f18249..01324cae05 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -90,9 +90,8 @@ set(FDBSERVER_SRCS
   QuietDatabase.actor.cpp
   QuietDatabase.h
   RadixTree.h
+  Ratekeeper.h
   Ratekeeper.actor.cpp
-  RatekeeperData.actor.cpp
-  RatekeeperData.h
   RatekeeperInterface.h
   RecoveryState.h
   RestoreCommon.actor.h
diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 76d8939a11..38564b3359 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -1,9 +1,9 @@
 /*
- * Ratekeeper.actor.cpp
+ * RatekeeperData.actor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,18 +18,1146 @@
  * limitations under the License.
  */
 
-#include "fdbserver/WorkerInterface.actor.h"
-#include "flow/IndexedSet.h"
-#include "fdbrpc/FailureMonitor.h"
-#include "fdbrpc/Smoother.h"
-#include "fdbrpc/simulator.h"
-#include "fdbclient/DatabaseContext.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/RatekeeperData.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/WaitFailure.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/Ratekeeper.h"
+#include "flow/actorcompiler.h" // must be last include
+
+const char* limitReasonName[] = { "workload",
+	                              "storage_server_write_queue_size",
+	                              "storage_server_write_bandwidth_mvcc",
+	                              "storage_server_readable_behind",
+	                              "log_server_mvcc_write_bandwidth",
+	                              "log_server_write_queue",
+	                              "storage_server_min_free_space",
+	                              "storage_server_min_free_space_ratio",
+	                              "log_server_min_free_space",
+	                              "log_server_min_free_space_ratio",
+	                              "storage_server_durability_lag",
+	                              "storage_server_list_fetch_failed" };
+static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
+
+int limitReasonEnd = limitReason_t_end;
+
+// NOTE: This has a corresponding table in Script.cs (see RatekeeperReason graph)
+// IF UPDATING THIS ARRAY, UPDATE SCRIPT.CS!
+const char* limitReasonDesc[] = { "Workload or read performance.",
+	                              "Storage server performance (storage queue).",
+	                              "Storage server MVCC memory.",
+	                              "Storage server version falling behind.",
+	                              "Log server MVCC memory.",
+	                              "Storage server performance (log queue).",
+	                              "Storage server running out of space (approaching 100MB limit).",
+	                              "Storage server running out of space (approaching 5% limit).",
+	                              "Log server running out of space (approaching 100MB limit).",
+	                              "Log server running out of space (approaching 5% limit).",
+	                              "Storage server durable version falling behind.",
+	                              "Unable to fetch storage server list." };
+
+static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
+
+ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
+	try {
+		wait(in);
+		return Void();
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && !errOut.isSet())
+			errOut.sendError(e);
+		throw;
+	}
+}
+
+class RatekeeperDataImpl {
+public:
+	ACTOR static Future<Void> configurationMonitor(RatekeeperData* self) {
+		loop {
+			state ReadYourWritesTransaction tr(self->db);
+
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+					ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+
+					self->configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
+
+					state Future<Void> watchFuture =
+					    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
+					    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
+					    tr.watch(failedLocalityVersionKey);
+					wait(tr.commit());
+					wait(watchFuture);
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> monitorServerListChange(
+	    RatekeeperData* self,
+	    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+		state std::map<UID, StorageServerInterface> oldServers;
+		state Transaction tr(self->db);
+
+		loop {
+			try {
+				if (now() - self->lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
+					TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", self->id)
+					    .detail("Latency", now() - self->lastSSListFetchedTimestamp);
+				}
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
+				    wait(getServerListAndProcessClasses(&tr));
+				self->lastSSListFetchedTimestamp = now();
+
+				std::map<UID, StorageServerInterface> newServers;
+				for (const auto& [ssi, _] : results) {
+					const UID serverId = ssi.id();
+					newServers[serverId] = ssi;
+
+					if (oldServers.count(serverId)) {
+						if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) {
+							serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
+						}
+						oldServers.erase(serverId);
+					} else {
+						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
+					}
+				}
+
+				for (const auto& it : oldServers) {
+					serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
+				}
+
+				oldServers.swap(newServers);
+				tr = Transaction(self->db);
+				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
+			} catch (Error& e) {
+				TraceEvent("RatekeeperGetSSListError", self->id).error(e).suppressFor(1.0);
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> trackStorageServerQueueInfo(RatekeeperData* self, StorageServerInterface ssi) {
+		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
+		state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
+		TraceEvent("RkTracking", self->id)
+		    .detail("StorageServer", ssi.id())
+		    .detail("Locality", ssi.locality.toString());
+		try {
+			loop {
+				ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
+				    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
+				if (reply.present()) {
+					myQueueInfo->value.valid = true;
+					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
+					myQueueInfo->value.lastReply = reply.get();
+					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
+						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
+						myQueueInfo->value.smoothDurableVersion.reset(reply.get().durableVersion);
+						myQueueInfo->value.smoothLatestVersion.reset(reply.get().version);
+					} else {
+						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
+						                                       myQueueInfo->value.prevReply.bytesDurable);
+						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
+						myQueueInfo->value.smoothDurableVersion.setTotal(reply.get().durableVersion);
+						myQueueInfo->value.smoothLatestVersion.setTotal(reply.get().version);
+					}
+
+					myQueueInfo->value.busiestReadTag = reply.get().busiestTag;
+					myQueueInfo->value.busiestReadTagFractionalBusyness = reply.get().busiestTagFractionalBusyness;
+					myQueueInfo->value.busiestReadTagRate = reply.get().busiestTagRate;
+				} else {
+					if (myQueueInfo->value.valid) {
+						TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id());
+					}
+					myQueueInfo->value.valid = false;
+				}
+
+				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+				     IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
+				                                                    FailureStatus(false)));
+			}
+		} catch (...) {
+			// including cancellation
+			self->storageQueueInfo.erase(myQueueInfo);
+			throw;
+		}
+	}
+
+	ACTOR static Future<Void> trackTLogQueueInfo(RatekeeperData* self, TLogInterface tli) {
+		self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
+		state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
+		TraceEvent("RkTracking", self->id).detail("TransactionLog", tli.id());
+		try {
+			loop {
+				ErrorOr<TLogQueuingMetricsReply> reply = wait(tli.getQueuingMetrics.getReplyUnlessFailedFor(
+				    TLogQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
+				if (reply.present()) {
+					myQueueInfo->value.valid = true;
+					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
+					myQueueInfo->value.lastReply = reply.get();
+					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
+						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
+					} else {
+						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
+						                                       myQueueInfo->value.prevReply.bytesDurable);
+						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
+						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
+						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
+						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
+					}
+				} else {
+					if (myQueueInfo->value.valid) {
+						TraceEvent("RkTLogDidNotRespond", self->id).detail("TransactionLog", tli.id());
+					}
+					myQueueInfo->value.valid = false;
+				}
+
+				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+				     IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
+				                                                    FailureStatus(false)));
+			}
+		} catch (...) {
+			// including cancellation
+			self->tlogQueueInfo.erase(myQueueInfo);
+			throw;
+		}
+	}
+
+	ACTOR static Future<Void> trackEachStorageServer(
+	    RatekeeperData* self,
+	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+		state Map<UID, Future<Void>> actors;
+		state Promise<Void> err;
+		loop choose {
+			when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
+				wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
+				if (change.second.present()) {
+					if (!change.second.get().isTss()) {
+						auto& a = actors[change.first];
+						a = Future<Void>();
+						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
+					}
+				} else
+					actors.erase(change.first);
+			}
+			when(wait(err.getFuture())) {}
+		}
+	}
+
+	ACTOR static Future<Void> monitorThrottlingChanges(RatekeeperData* self) {
+		state bool committed = false;
+		loop {
+			state ReadYourWritesTransaction tr(self->db);
+
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
+					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
+
+					if (!committed) {
+						BinaryWriter limitWriter(Unversioned());
+						limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
+						tr.set(tagThrottleLimitKey, limitWriter.toValue());
+					}
+
+					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
+
+					if (autoThrottlingEnabled.get().present() &&
+					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
+						TEST(true); // Auto-throttling disabled
+						if (self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
+						}
+						self->autoThrottlingEnabled = false;
+					} else if (autoThrottlingEnabled.get().present() &&
+					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
+						TEST(true); // Auto-throttling enabled
+						if (!self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
+						}
+						self->autoThrottlingEnabled = true;
+					} else {
+						TEST(true); // Auto-throttling unspecified
+						if (autoThrottlingEnabled.get().present()) {
+							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
+							    .detail("Value", autoThrottlingEnabled.get().get());
+						}
+						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
+						if (!committed)
+							tr.set(tagThrottleAutoEnabledKey,
+							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
+					}
+
+					RkTagThrottleCollection updatedTagThrottles;
+
+					TraceEvent("RatekeeperReadThrottledTags", self->id)
+					    .detail("NumThrottledTags", throttledTagKeys.get().size());
+					for (auto entry : throttledTagKeys.get()) {
+						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
+						TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
+
+						ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
+
+						if (tagValue.expirationTime == 0 ||
+						    tagValue.expirationTime > now() + tagValue.initialDuration) {
+							TEST(true); // Converting tag throttle duration to absolute time
+							tagValue.expirationTime = now() + tagValue.initialDuration;
+							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
+							wr << tagValue;
+							state Value value = wr.toValue();
+
+							tr.set(entry.key, value);
+						}
+
+						if (tagValue.expirationTime > now()) {
+							TransactionTag tag = *tagKey.tags.begin();
+							Optional<ClientTagThrottleLimits> oldLimits =
+							    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
+
+							if (tagKey.throttleType == TagThrottleType::AUTO) {
+								updatedTagThrottles.autoThrottleTag(
+								    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
+								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
+									updatedTagThrottles.busyReadTagCount++;
+								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
+									updatedTagThrottles.busyWriteTagCount++;
+								}
+							} else {
+								updatedTagThrottles.manualThrottleTag(self->id,
+								                                      tag,
+								                                      tagKey.priority,
+								                                      tagValue.tpsRate,
+								                                      tagValue.expirationTime,
+								                                      oldLimits);
+							}
+						}
+					}
+
+					self->throttledTags = std::move(updatedTagThrottles);
+					++self->throttledTagChangeId;
+
+					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
+					wait(tr.commit());
+					committed = true;
+
+					wait(watchFuture);
+					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
+					TEST(true); // Tag throttle changes detected
+					break;
+				} catch (Error& e) {
+					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+		state RatekeeperData self(rkInterf.id(),
+		                          openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
+		state Future<Void> timeout = Void();
+		state std::vector<Future<Void>> tlogTrackers;
+		state std::vector<TLogInterface> tlogInterfs;
+		state Promise<Void> err;
+		state Future<Void> collection = actorCollection(self.addActor.getFuture());
+
+		TraceEvent("RatekeeperStarting", rkInterf.id());
+		self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
+		self.addActor.send(self.configurationMonitor());
+
+		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
+		self.addActor.send(self.monitorServerListChange(serverChanges));
+		self.addActor.send(self.trackEachStorageServer(serverChanges.getFuture()));
+		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
+
+		self.addActor.send(self.monitorThrottlingChanges());
+		RatekeeperData* selfPtr = &self; // let flow compiler capture self
+		self.addActor.send(recurring([selfPtr]() { selfPtr->refreshStorageServerCommitCost(); },
+		                             SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
+
+		TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
+		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
+		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
+		    .detail(
+		        "Rate",
+		        (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
+		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+		             2.0));
+
+		TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
+		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
+		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
+		    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
+		    .detail(
+		        "Rate",
+		        (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
+		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+		             2.0));
+
+		tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+		tlogTrackers.reserve(tlogInterfs.size());
+		for (int i = 0; i < tlogInterfs.size(); i++) {
+			tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
+		}
+
+		self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+
+		try {
+			state bool lastLimited = false;
+			loop choose {
+				when(wait(timeout)) {
+					self.updateRate(&self.normalLimits);
+					self.updateRate(&self.batchLimits);
+
+					lastLimited = self.smoothReleasedTransactions.smoothRate() >
+					              SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
+					double tooOld = now() - 1.0;
+					for (auto p = self.grvProxyInfo.begin(); p != self.grvProxyInfo.end();) {
+						if (p->second.lastUpdateTime < tooOld)
+							p = self.grvProxyInfo.erase(p);
+						else
+							++p;
+					}
+					timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
+				}
+				when(GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
+					GetRateInfoReply reply;
+
+					auto& p = self.grvProxyInfo[req.requesterID];
+					//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
+					if (p.totalTransactions > 0) {
+						self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
+
+						for (auto tag : req.throttledTagCounts) {
+							self.throttledTags.addRequests(tag.first, tag.second);
+						}
+					}
+					if (p.batchTransactions > 0) {
+						self.smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions -
+						                                              p.batchTransactions);
+					}
+
+					p.totalTransactions = req.totalReleasedTransactions;
+					p.batchTransactions = req.batchReleasedTransactions;
+					p.lastUpdateTime = now();
+
+					reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
+					reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
+					reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
+
+					if (p.lastThrottledTagChangeId != self.throttledTagChangeId ||
+					    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
+						p.lastThrottledTagChangeId = self.throttledTagChangeId;
+						p.lastTagPushTime = now();
+
+						reply.throttledTags = self.throttledTags.getClientRates(self.autoThrottlingEnabled);
+						bool returningTagsToProxy =
+						    reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
+						TEST(returningTagsToProxy); // Returning tag throttles to a proxy
+					}
+
+					reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
+					reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
+					reply.healthMetrics.batchLimited = lastLimited;
+
+					req.reply.send(reply);
+				}
+				when(HaltRatekeeperRequest req = waitNext(rkInterf.haltRatekeeper.getFuture())) {
+					req.reply.send(Void());
+					TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
+					break;
+				}
+				when(ReportCommitCostEstimationRequest req =
+				         waitNext(rkInterf.reportCommitCostEstimation.getFuture())) {
+					self.updateCommitCostEstimation(req.ssTrTagCommitCost);
+					req.reply.send(Void());
+				}
+				when(wait(err.getFuture())) {}
+				when(wait(dbInfo->onChange())) {
+					if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
+						tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+						tlogTrackers = std::vector<Future<Void>>();
+						for (int i = 0; i < tlogInterfs.size(); i++)
+							tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
+					}
+					self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+				}
+				when(wait(collection)) {
+					ASSERT(false);
+					throw internal_error();
+				}
+			}
+		} catch (Error& err) {
+			TraceEvent("RatekeeperDied", rkInterf.id()).error(err, true);
+		}
+		return Void();
+	}
+
+}; // class RatekeeperDataImpl
+
+Future<Void> RatekeeperData::configurationMonitor() {
+	return RatekeeperDataImpl::configurationMonitor(this);
+}
+
+Future<Void> RatekeeperData::monitorServerListChange(
+    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	return RatekeeperDataImpl::monitorServerListChange(this, serverChanges);
+}
+
+Future<Void> RatekeeperData::trackEachStorageServer(
+    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	return RatekeeperDataImpl::trackEachStorageServer(this, serverChanges);
+}
+
+Future<Void> RatekeeperData::trackStorageServerQueueInfo(StorageServerInterface ssi) {
+	return RatekeeperDataImpl::trackStorageServerQueueInfo(this, ssi);
+}
+
+Future<Void> RatekeeperData::trackTLogQueueInfo(TLogInterface tli) {
+	return RatekeeperDataImpl::trackTLogQueueInfo(this, tli);
+}
+
+Future<Void> RatekeeperData::monitorThrottlingChanges() {
+	return RatekeeperDataImpl::monitorThrottlingChanges(this);
+}
+
+Future<Void> RatekeeperData::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	return RatekeeperDataImpl::run(rkInterf, dbInfo);
+}
+
+RatekeeperData::RatekeeperData(UID id, Database db)
+  : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
+    actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
+    lastBusiestCommitTagPick(0), throttledTagChangeId(0), normalLimits(TransactionPriority::DEFAULT,
+                                                                       "",
+                                                                       SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
+                                                                       SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
+                                                                       SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
+                                                                       SERVER_KNOBS->SPRING_BYTES_TLOG,
+                                                                       SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
+                                                                       SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
+    batchLimits(TransactionPriority::BATCH,
+                "Batch",
+                SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
+                SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH,
+                SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
+                SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
+                SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
+    autoThrottlingEnabled(false) {
+	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
+	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
+}
+
+void RatekeeperData::updateCommitCostEstimation(
+    UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation) {
+	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
+		auto tagCostIt = costEstimation.find(it->key);
+		if (tagCostIt == costEstimation.end())
+			continue;
+		for (const auto& [tagName, cost] : tagCostIt->second) {
+			it->value.tagCostEst[tagName] += cost;
+			it->value.totalWriteCosts += cost.getCostSum();
+			it->value.totalWriteOps += cost.getOpsSum();
+		}
+	}
+}
+
+void RatekeeperData::updateRate(RatekeeperLimits* limits) {
+	// double controlFactor = ;  // dt / eFoldingTime
+
+	double actualTps = smoothReleasedTransactions.smoothRate();
+	actualTpsMetric = (int64_t)actualTps;
+	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this
+	// value
+	actualTps =
+	    std::max(std::max(1.0, actualTps), smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
+
+	if (actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
+		actualTpsHistory.pop_front();
+	}
+	actualTpsHistory.push_back(actualTps);
+
+	limits->tpsLimit = std::numeric_limits<double>::infinity();
+	UID reasonID = UID();
+	limitReason_t limitReason = limitReason_t::unlimited;
+
+	int sscount = 0;
+
+	int64_t worstFreeSpaceStorageServer = std::numeric_limits<int64_t>::max();
+	int64_t worstStorageQueueStorageServer = 0;
+	int64_t limitingStorageQueueStorageServer = 0;
+	int64_t worstDurabilityLag = 0;
+
+	std::multimap<double, StorageQueueInfo*> storageTpsLimitReverseIndex;
+	std::multimap<int64_t, StorageQueueInfo*> storageDurabilityLagReverseIndex;
+
+	std::map<UID, limitReason_t> ssReasons;
+
+	bool printRateKeepLimitReasonDetails =
+	    SERVER_KNOBS->RATEKEEPER_PRINT_LIMIT_REASON &&
+	    (deterministicRandom()->random01() < SERVER_KNOBS->RATEKEEPER_LIMIT_REASON_SAMPLE_RATE);
+
+	// Look at each storage server's write queue and local rate, compute and store the desired rate ratio
+	for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) {
+		auto& ss = i->value;
+		if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
+			continue;
+		++sscount;
+
+		limitReason_t ssLimitReason = limitReason_t::unlimited;
+
+		int64_t minFreeSpace =
+		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
+		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * ss.smoothTotalSpace.smoothTotal()));
+
+		worstFreeSpaceStorageServer =
+		    std::min(worstFreeSpaceStorageServer, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace);
+
+		int64_t springBytes = std::max<int64_t>(
+		    1, std::min<int64_t>(limits->storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
+		int64_t targetBytes = std::max<int64_t>(
+		    1, std::min(limits->storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
+		if (targetBytes != limits->storageTargetBytes) {
+			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
+				ssLimitReason = limitReason_t::storage_server_min_free_space;
+			} else {
+				ssLimitReason = limitReason_t::storage_server_min_free_space_ratio;
+			}
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("Reason", ssLimitReason)
+				    .detail("SSID", ss.id)
+				    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
+				    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
+				    .detail("TargetBytes", targetBytes)
+				    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
+				    .detail("MinFreeSpace", minFreeSpace);
+			}
+		}
+
+		int64_t storageQueue = ss.lastReply.bytesInput - ss.smoothDurableBytes.smoothTotal();
+		worstStorageQueueStorageServer = std::max(worstStorageQueueStorageServer, storageQueue);
+
+		int64_t storageDurabilityLag = ss.smoothLatestVersion.smoothTotal() - ss.smoothDurableVersion.smoothTotal();
+		worstDurabilityLag = std::max(worstDurabilityLag, storageDurabilityLag);
+
+		storageDurabilityLagReverseIndex.insert(std::make_pair(-1 * storageDurabilityLag, &ss));
+
+		auto& ssMetrics = healthMetrics.storageStats[ss.id];
+		ssMetrics.storageQueue = storageQueue;
+		ssMetrics.storageDurabilityLag = storageDurabilityLag;
+		ssMetrics.cpuUsage = ss.lastReply.cpuUsage;
+		ssMetrics.diskUsage = ss.lastReply.diskUsage;
+
+		double targetRateRatio = std::min((storageQueue - targetBytes + springBytes) / (double)springBytes, 2.0);
+
+		if (limits->priority == TransactionPriority::DEFAULT) {
+			tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag);
+		}
+
+		double inputRate = ss.smoothInputBytes.smoothRate();
+		// inputRate = std::max( inputRate, actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE );
+
+		/*if( deterministicRandom()->random01() < 0.1 ) {
+		  std::string name = "RatekeeperUpdateRate" + limits.context;
+		  TraceEvent(name, ss.id)
+		  .detail("MinFreeSpace", minFreeSpace)
+		  .detail("SpringBytes", springBytes)
+		  .detail("TargetBytes", targetBytes)
+		  .detail("SmoothTotalSpaceTotal", ss.smoothTotalSpace.smoothTotal())
+		  .detail("SmoothFreeSpaceTotal", ss.smoothFreeSpace.smoothTotal())
+		  .detail("LastReplyBytesInput", ss.lastReply.bytesInput)
+		  .detail("SmoothDurableBytesTotal", ss.smoothDurableBytes.smoothTotal())
+		  .detail("TargetRateRatio", targetRateRatio)
+		  .detail("SmoothInputBytesRate", ss.smoothInputBytes.smoothRate())
+		  .detail("ActualTPS", actualTps)
+		  .detail("InputRate", inputRate)
+		  .detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate())
+		  .detail("B", b);
+		  }*/
+
+		// Don't let any storage server use up its target bytes faster than its MVCC window!
+		double maxBytesPerSecond =
+		    (targetBytes - springBytes) /
+		    ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0);
+		double limitTps = std::min(actualTps * maxBytesPerSecond / std::max(1.0e-8, inputRate),
+		                           maxBytesPerSecond * SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+		if (ssLimitReason == limitReason_t::unlimited)
+			ssLimitReason = limitReason_t::storage_server_write_bandwidth_mvcc;
+
+		if (targetRateRatio > 0 && inputRate > 0) {
+			ASSERT(inputRate != 0);
+			double smoothedRate =
+			    std::max(ss.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+			double x = smoothedRate / (inputRate * targetRateRatio);
+			double lim = actualTps * x;
+			if (lim < limitTps) {
+				double oldLimitTps = limitTps;
+				limitTps = lim;
+				if (ssLimitReason == limitReason_t::unlimited ||
+				    ssLimitReason == limitReason_t::storage_server_write_bandwidth_mvcc) {
+					if (printRateKeepLimitReasonDetails) {
+						TraceEvent("RatekeeperLimitReasonDetails")
+						    .detail("Reason", limitReason_t::storage_server_write_queue_size)
+						    .detail("FromReason", ssLimitReason)
+						    .detail("SSID", ss.id)
+						    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
+						    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
+						    .detail("LimitsStorageSpringBytes", limits->storageSpringBytes)
+						    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
+						    .detail("MinFreeSpace", minFreeSpace)
+						    .detail("SSLastReplyBytesInput", ss.lastReply.bytesInput)
+						    .detail("SSSmoothDurableBytes", ss.smoothDurableBytes.smoothTotal())
+						    .detail("StorageQueue", storageQueue)
+						    .detail("TargetBytes", targetBytes)
+						    .detail("SpringBytes", springBytes)
+						    .detail("SSVerySmoothDurableBytesSmoothRate", ss.verySmoothDurableBytes.smoothRate())
+						    .detail("SmoothedRate", smoothedRate)
+						    .detail("X", x)
+						    .detail("ActualTps", actualTps)
+						    .detail("Lim", lim)
+						    .detail("LimitTps", oldLimitTps)
+						    .detail("InputRate", inputRate)
+						    .detail("TargetRateRatio", targetRateRatio);
+					}
+					ssLimitReason = limitReason_t::storage_server_write_queue_size;
+				}
+			}
+		}
+
+		storageTpsLimitReverseIndex.insert(std::make_pair(limitTps, &ss));
+
+		if (limitTps < limits->tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space ||
+		                                    ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
+			reasonID = ss.id;
+			limits->tpsLimit = limitTps;
+			limitReason = ssLimitReason;
+		}
+
+		ssReasons[ss.id] = ssLimitReason;
+	}
+
+	std::set<Optional<Standalone<StringRef>>> ignoredMachines;
+	for (auto ss = storageTpsLimitReverseIndex.begin();
+	     ss != storageTpsLimitReverseIndex.end() && ss->first < limits->tpsLimit;
+	     ++ss) {
+		if (ignoredMachines.size() <
+		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
+			ignoredMachines.insert(ss->second->locality.zoneId());
+			continue;
+		}
+		if (ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
+			continue;
+		}
+
+		limitingStorageQueueStorageServer =
+		    ss->second->lastReply.bytesInput - ss->second->smoothDurableBytes.smoothTotal();
+		limits->tpsLimit = ss->first;
+		reasonID = storageTpsLimitReverseIndex.begin()->second->id; // Although we aren't controlling based on the worst
+		// SS, we still report it as the limiting process
+		limitReason = ssReasons[reasonID];
+		break;
+	}
+
+	// Calculate limited durability lag
+	int64_t limitingDurabilityLag = 0;
+
+	std::set<Optional<Standalone<StringRef>>> ignoredDurabilityLagMachines;
+	for (auto ss = storageDurabilityLagReverseIndex.begin(); ss != storageDurabilityLagReverseIndex.end(); ++ss) {
+		if (ignoredDurabilityLagMachines.size() <
+		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
+			ignoredDurabilityLagMachines.insert(ss->second->locality.zoneId());
+			continue;
+		}
+		if (ignoredDurabilityLagMachines.count(ss->second->locality.zoneId()) > 0) {
+			continue;
+		}
+
+		limitingDurabilityLag = -1 * ss->first;
+		if (limitingDurabilityLag > limits->durabilityLagTargetVersions &&
+		    actualTpsHistory.size() > SERVER_KNOBS->NEEDED_TPS_HISTORY_SAMPLES) {
+			if (limits->durabilityLagLimit == std::numeric_limits<double>::infinity()) {
+				double maxTps = 0;
+				for (int i = 0; i < actualTpsHistory.size(); i++) {
+					maxTps = std::max(maxTps, actualTpsHistory[i]);
+				}
+				limits->durabilityLagLimit = SERVER_KNOBS->INITIAL_DURABILITY_LAG_MULTIPLIER * maxTps;
+			}
+			if (limitingDurabilityLag > limits->lastDurabilityLag) {
+				limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_REDUCTION_RATE * limits->durabilityLagLimit;
+			}
+			if (limits->durabilityLagLimit < limits->tpsLimit) {
+				if (printRateKeepLimitReasonDetails) {
+					TraceEvent("RatekeeperLimitReasonDetails")
+					    .detail("SSID", ss->second->id)
+					    .detail("Reason", limitReason_t::storage_server_durability_lag)
+					    .detail("LimitsDurabilityLagLimit", limits->durabilityLagLimit)
+					    .detail("LimitsTpsLimit", limits->tpsLimit)
+					    .detail("LimitingDurabilityLag", limitingDurabilityLag)
+					    .detail("LimitsLastDurabilityLag", limits->lastDurabilityLag);
+				}
+				limits->tpsLimit = limits->durabilityLagLimit;
+				limitReason = limitReason_t::storage_server_durability_lag;
+			}
+		} else if (limits->durabilityLagLimit != std::numeric_limits<double>::infinity() &&
+		           limitingDurabilityLag >
+		               limits->durabilityLagTargetVersions - SERVER_KNOBS->DURABILITY_LAG_UNLIMITED_THRESHOLD) {
+			limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_INCREASE_RATE * limits->durabilityLagLimit;
+		} else {
+			limits->durabilityLagLimit = std::numeric_limits<double>::infinity();
+		}
+		limits->lastDurabilityLag = limitingDurabilityLag;
+		break;
+	}
+
+	healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
+	healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
+	healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
+	healthMetrics.limitingStorageDurabilityLag = limitingDurabilityLag;
+
+	double writeToReadLatencyLimit = 0;
+	Version worstVersionLag = 0;
+	Version limitingVersionLag = 0;
+
+	{
+		Version minSSVer = std::numeric_limits<Version>::max();
+		Version minLimitingSSVer = std::numeric_limits<Version>::max();
+		for (const auto& it : storageQueueInfo) {
+			auto& ss = it.value;
+			if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
+				continue;
+
+			minSSVer = std::min(minSSVer, ss.lastReply.version);
+
+			// Machines that ratekeeper isn't controlling can fall arbitrarily far behind
+			if (ignoredMachines.count(it.value.locality.zoneId()) == 0) {
+				minLimitingSSVer = std::min(minLimitingSSVer, ss.lastReply.version);
+			}
+		}
+
+		Version maxTLVer = std::numeric_limits<Version>::min();
+		for (const auto& it : tlogQueueInfo) {
+			auto& tl = it.value;
+			if (!tl.valid)
+				continue;
+			maxTLVer = std::max(maxTLVer, tl.lastReply.v);
+		}
+
+		if (minSSVer != std::numeric_limits<Version>::max() && maxTLVer != std::numeric_limits<Version>::min()) {
+			// writeToReadLatencyLimit: 0 = infinte speed; 1 = TL durable speed ; 2 = half TL durable speed
+			writeToReadLatencyLimit =
+			    ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference / 2) / (limits->maxVersionDifference / 4);
+			worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
+			limitingVersionLag = std::max((Version)0, maxTLVer - minLimitingSSVer);
+		}
+	}
+
+	int64_t worstFreeSpaceTLog = std::numeric_limits<int64_t>::max();
+	int64_t worstStorageQueueTLog = 0;
+	int tlcount = 0;
+	for (auto& it : tlogQueueInfo) {
+		auto& tl = it.value;
+		if (!tl.valid)
+			continue;
+		++tlcount;
+
+		limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue;
+
+		int64_t minFreeSpace =
+		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
+		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.smoothTotalSpace.smoothTotal()));
+
+		worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace);
+
+		int64_t springBytes = std::max<int64_t>(
+		    1, std::min<int64_t>(limits->logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
+		int64_t targetBytes = std::max<int64_t>(
+		    1, std::min(limits->logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
+		if (targetBytes != limits->logTargetBytes) {
+			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
+				tlogLimitReason = limitReason_t::log_server_min_free_space;
+			} else {
+				tlogLimitReason = limitReason_t::log_server_min_free_space_ratio;
+			}
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("TLogID", tl.id)
+				    .detail("Reason", tlogLimitReason)
+				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+				    .detail("TargetBytes", targetBytes)
+				    .detail("MinFreeSpace", minFreeSpace);
+			}
+		}
+
+		int64_t queue = tl.lastReply.bytesInput - tl.smoothDurableBytes.smoothTotal();
+		healthMetrics.tLogQueue[tl.id] = queue;
+		int64_t b = queue - targetBytes;
+		worstStorageQueueTLog = std::max(worstStorageQueueTLog, queue);
+
+		if (tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2) {
+			if (now() - lastWarning > 5.0) {
+				lastWarning = now();
+				TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero", id).detail("ReasonId", tl.id);
+			}
+			reasonID = tl.id;
+			limitReason = limitReason_t::log_server_min_free_space;
+			limits->tpsLimit = 0.0;
+		}
+
+		double targetRateRatio = std::min((b + springBytes) / (double)springBytes, 2.0);
+
+		if (writeToReadLatencyLimit > targetRateRatio) {
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("TLogID", tl.id)
+				    .detail("Reason", limitReason_t::storage_server_readable_behind)
+				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+				    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
+				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+				    .detail("SpringBytes", springBytes)
+				    .detail("TargetBytes", targetBytes)
+				    .detail("TLLastReplyBytesInput", tl.lastReply.bytesInput)
+				    .detail("TLSmoothDurableBytes", tl.smoothDurableBytes.smoothTotal())
+				    .detail("Queue", queue)
+				    .detail("B", b)
+				    .detail("TargetRateRatio", targetRateRatio)
+				    .detail("WriteToReadLatencyLimit", writeToReadLatencyLimit)
+				    .detail("MinFreeSpace", minFreeSpace)
+				    .detail("LimitsMaxVersionDifference", limits->maxVersionDifference);
+			}
+			targetRateRatio = writeToReadLatencyLimit;
+			tlogLimitReason = limitReason_t::storage_server_readable_behind;
+		}
+
+		double inputRate = tl.smoothInputBytes.smoothRate();
+
+		if (targetRateRatio > 0) {
+			double smoothedRate =
+			    std::max(tl.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
+			double x = smoothedRate / (inputRate * targetRateRatio);
+			if (targetRateRatio < .75) //< FIXME: KNOB for 2.0
+				x = std::max(x, 0.95);
+			double lim = actualTps * x;
+			if (lim < limits->tpsLimit) {
+				limits->tpsLimit = lim;
+				reasonID = tl.id;
+				limitReason = tlogLimitReason;
+			}
+		}
+		if (inputRate > 0) {
+			// Don't let any tlogs use up its target bytes faster than its MVCC window!
+			double x =
+			    ((targetBytes - springBytes) /
+			     ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+			      2.0)) /
+			    inputRate;
+			double lim = actualTps * x;
+			if (lim < limits->tpsLimit) {
+				if (printRateKeepLimitReasonDetails) {
+					TraceEvent("RatekeeperLimitReasonDetails")
+					    .detail("Reason", limitReason_t::log_server_mvcc_write_bandwidth)
+					    .detail("TLogID", tl.id)
+					    .detail("MinFreeSpace", minFreeSpace)
+					    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
+					    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
+					    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
+					    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+					    .detail("SpringBytes", springBytes)
+					    .detail("TargetBytes", targetBytes)
+					    .detail("InputRate", inputRate)
+					    .detail("X", x)
+					    .detail("ActualTps", actualTps)
+					    .detail("Lim", lim)
+					    .detail("LimitsTpsLimit", limits->tpsLimit);
+				}
+				limits->tpsLimit = lim;
+				reasonID = tl.id;
+				limitReason = limitReason_t::log_server_mvcc_write_bandwidth;
+			}
+		}
+	}
+
+	healthMetrics.worstTLogQueue = worstStorageQueueTLog;
+
+	limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
+
+	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+		limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
+	}
+
+	int64_t totalDiskUsageBytes = 0;
+	for (auto& t : tlogQueueInfo) {
+		if (t.value.valid) {
+			totalDiskUsageBytes += t.value.lastReply.storageBytes.used;
+		}
+	}
+	for (auto& s : storageQueueInfo) {
+		if (s.value.valid) {
+			totalDiskUsageBytes += s.value.lastReply.storageBytes.used;
+		}
+	}
+
+	if (now() - lastSSListFetchedTimestamp > SERVER_KNOBS->STORAGE_SERVER_LIST_FETCH_TIMEOUT) {
+		limits->tpsLimit = 0.0;
+		limitReason = limitReason_t::storage_server_list_fetch_failed;
+		reasonID = UID();
+		TraceEvent(SevWarnAlways, "RkSSListFetchTimeout", id).suppressFor(1.0);
+	} else if (limits->tpsLimit == std::numeric_limits<double>::infinity()) {
+		limits->tpsLimit = SERVER_KNOBS->RATEKEEPER_DEFAULT_LIMIT;
+	}
+
+	limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
+	limits->reasonMetric = limitReason;
+
+	if (deterministicRandom()->random01() < 0.1) {
+		const std::string& name = limits->rkUpdateEventCacheHolder.getPtr()->trackingKey;
+		TraceEvent(name.c_str(), id)
+		    .detail("TPSLimit", limits->tpsLimit)
+		    .detail("Reason", limitReason)
+		    .detail("ReasonServerID", reasonID == UID() ? std::string() : Traceable<UID>::toString(reasonID))
+		    .detail("ReleasedTPS", smoothReleasedTransactions.smoothRate())
+		    .detail("ReleasedBatchTPS", smoothBatchReleasedTransactions.smoothRate())
+		    .detail("TPSBasis", actualTps)
+		    .detail("StorageServers", sscount)
+		    .detail("GrvProxies", grvProxyInfo.size())
+		    .detail("TLogs", tlcount)
+		    .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer)
+		    .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog)
+		    .detail("WorstStorageServerQueue", worstStorageQueueStorageServer)
+		    .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer)
+		    .detail("WorstTLogQueue", worstStorageQueueTLog)
+		    .detail("TotalDiskUsageBytes", totalDiskUsageBytes)
+		    .detail("WorstStorageServerVersionLag", worstVersionLag)
+		    .detail("LimitingStorageServerVersionLag", limitingVersionLag)
+		    .detail("WorstStorageServerDurabilityLag", worstDurabilityLag)
+		    .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag)
+		    .detail("TagsAutoThrottled", throttledTags.autoThrottleCount())
+		    .detail("TagsAutoThrottledBusyRead", throttledTags.busyReadTagCount)
+		    .detail("TagsAutoThrottledBusyWrite", throttledTags.busyWriteTagCount)
+		    .detail("TagsManuallyThrottled", throttledTags.manualThrottleCount())
+		    .detail("AutoThrottlingEnabled", autoThrottlingEnabled)
+		    .trackLatest(name);
+	}
+}
+
+Future<Void> RatekeeperData::refreshStorageServerCommitCost() {
+	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
+		lastBusiestCommitTagPick = now();
+		return Void();
+	}
+	double elapsed = now() - lastBusiestCommitTagPick;
+	// for each SS, select the busiest commit tag from ssTrTagCommitCost
+	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
+		it->value.busiestWriteTag.reset();
+		TransactionTag busiestTag;
+		TransactionCommitCostEstimation maxCost;
+		double maxRate = 0, maxBusyness = 0;
+		for (const auto& [tag, cost] : it->value.tagCostEst) {
+			double rate = cost.getCostSum() / elapsed;
+			if (rate > maxRate) {
+				busiestTag = tag;
+				maxRate = rate;
+				maxCost = cost;
+			}
+		}
+		if (maxRate > SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE) {
+			it->value.busiestWriteTag = busiestTag;
+			// TraceEvent("RefreshSSCommitCost").detail("TotalWriteCost", it->value.totalWriteCost).detail("TotalWriteOps",it->value.totalWriteOps);
+			ASSERT(it->value.totalWriteCosts > 0);
+			maxBusyness = double(maxCost.getCostSum()) / it->value.totalWriteCosts;
+			it->value.busiestWriteTagFractionalBusyness = maxBusyness;
+			it->value.busiestWriteTagRate = maxRate;
+		}
+
+		TraceEvent("BusiestWriteTag", it->key)
+		    .detail("Elapsed", elapsed)
+		    .detail("Tag", printable(busiestTag))
+		    .detail("TagOps", maxCost.getOpsSum())
+		    .detail("TagCost", maxCost.getCostSum())
+		    .detail("TotalCost", it->value.totalWriteCosts)
+		    .detail("Reported", it->value.busiestWriteTag.present())
+		    .trackLatest(it->value.busiestWriteTagEventHolder->trackingKey);
+
+		// reset statistics
+		it->value.tagCostEst.clear();
+		it->value.totalWriteOps = 0;
+		it->value.totalWriteCosts = 0;
+	}
+	lastBusiestCommitTagPick = now();
+	return Void();
+}
+
+void RatekeeperData::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
+	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
+	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
+	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
+		TEST(true); // Transaction tag auto-throttled
+		Optional<double> clientRate = throttledTags.autoThrottleTag(id, tag, busyness);
+		if (clientRate.present()) {
+			TagSet tags;
+			tags.addTag(tag);
+
+			Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(db.getPtr());
+			addActor.send(ThrottleApi::throttleTags(dbRef,
+			                                        tags,
+			                                        clientRate.get(),
+			                                        SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+			                                        TagThrottleType::AUTO,
+			                                        TransactionPriority::DEFAULT,
+			                                        now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+			                                        reason));
+		}
+	}
+}
+
+void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
+	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
+	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
+	// future
+	if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
+	    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
+		if (ss.busiestWriteTag.present()) {
+			tryAutoThrottleTag(ss.busiestWriteTag.get(),
+			                   ss.busiestWriteTagRate,
+			                   ss.busiestWriteTagFractionalBusyness,
+			                   TagThrottledReason::BUSY_WRITE);
+		}
+		if (ss.busiestReadTag.present()) {
+			tryAutoThrottleTag(ss.busiestReadTag.get(),
+			                   ss.busiestReadTagRate,
+			                   ss.busiestReadTagFractionalBusyness,
+			                   TagThrottledReason::BUSY_READ);
+		}
+	}
+}
+
+ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	wait(RatekeeperData::run(rkInterf, dbInfo));
+	return Void();
+}
diff --git a/fdbserver/RatekeeperData.h b/fdbserver/Ratekeeper.h
similarity index 100%
rename from fdbserver/RatekeeperData.h
rename to fdbserver/Ratekeeper.h
diff --git a/fdbserver/RatekeeperData.actor.cpp b/fdbserver/RatekeeperData.actor.cpp
deleted file mode 100644
index e644240a16..0000000000
--- a/fdbserver/RatekeeperData.actor.cpp
+++ /dev/null
@@ -1,1163 +0,0 @@
-/*
- * RatekeeperData.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/RatekeeperData.h"
-#include "flow/actorcompiler.h" // must be last include
-
-const char* limitReasonName[] = { "workload",
-	                              "storage_server_write_queue_size",
-	                              "storage_server_write_bandwidth_mvcc",
-	                              "storage_server_readable_behind",
-	                              "log_server_mvcc_write_bandwidth",
-	                              "log_server_write_queue",
-	                              "storage_server_min_free_space",
-	                              "storage_server_min_free_space_ratio",
-	                              "log_server_min_free_space",
-	                              "log_server_min_free_space_ratio",
-	                              "storage_server_durability_lag",
-	                              "storage_server_list_fetch_failed" };
-static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-int limitReasonEnd = limitReason_t_end;
-
-// NOTE: This has a corresponding table in Script.cs (see RatekeeperReason graph)
-// IF UPDATING THIS ARRAY, UPDATE SCRIPT.CS!
-const char* limitReasonDesc[] = { "Workload or read performance.",
-	                              "Storage server performance (storage queue).",
-	                              "Storage server MVCC memory.",
-	                              "Storage server version falling behind.",
-	                              "Log server MVCC memory.",
-	                              "Storage server performance (log queue).",
-	                              "Storage server running out of space (approaching 100MB limit).",
-	                              "Storage server running out of space (approaching 5% limit).",
-	                              "Log server running out of space (approaching 100MB limit).",
-	                              "Log server running out of space (approaching 5% limit).",
-	                              "Storage server durable version falling behind.",
-	                              "Unable to fetch storage server list." };
-
-static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
-	try {
-		wait(in);
-		return Void();
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && !errOut.isSet())
-			errOut.sendError(e);
-		throw;
-	}
-}
-
-class RatekeeperDataImpl {
-public:
-	ACTOR static Future<Void> configurationMonitor(RatekeeperData* self) {
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
-
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-					RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
-					ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
-
-					self->configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
-
-					state Future<Void> watchFuture =
-					    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
-					    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
-					    tr.watch(failedLocalityVersionKey);
-					wait(tr.commit());
-					wait(watchFuture);
-					break;
-				} catch (Error& e) {
-					wait(tr.onError(e));
-				}
-			}
-		}
-	}
-
-	ACTOR static Future<Void> monitorServerListChange(
-	    RatekeeperData* self,
-	    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-		state std::map<UID, StorageServerInterface> oldServers;
-		state Transaction tr(self->db);
-
-		loop {
-			try {
-				if (now() - self->lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
-					TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", self->id)
-					    .detail("Latency", now() - self->lastSSListFetchedTimestamp);
-				}
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-				    wait(getServerListAndProcessClasses(&tr));
-				self->lastSSListFetchedTimestamp = now();
-
-				std::map<UID, StorageServerInterface> newServers;
-				for (const auto& [ssi, _] : results) {
-					const UID serverId = ssi.id();
-					newServers[serverId] = ssi;
-
-					if (oldServers.count(serverId)) {
-						if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) {
-							serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-						}
-						oldServers.erase(serverId);
-					} else {
-						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-					}
-				}
-
-				for (const auto& it : oldServers) {
-					serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
-				}
-
-				oldServers.swap(newServers);
-				tr = Transaction(self->db);
-				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
-			} catch (Error& e) {
-				TraceEvent("RatekeeperGetSSListError", self->id).error(e).suppressFor(1.0);
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> trackStorageServerQueueInfo(RatekeeperData* self, StorageServerInterface ssi) {
-		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
-		state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
-		TraceEvent("RkTracking", self->id)
-		    .detail("StorageServer", ssi.id())
-		    .detail("Locality", ssi.locality.toString());
-		try {
-			loop {
-				ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
-				    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-				if (reply.present()) {
-					myQueueInfo->value.valid = true;
-					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
-					myQueueInfo->value.lastReply = reply.get();
-					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
-						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
-						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
-						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
-						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
-						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
-						myQueueInfo->value.smoothDurableVersion.reset(reply.get().durableVersion);
-						myQueueInfo->value.smoothLatestVersion.reset(reply.get().version);
-					} else {
-						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
-						                                       myQueueInfo->value.prevReply.bytesDurable);
-						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
-						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
-						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
-						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
-						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
-						myQueueInfo->value.smoothDurableVersion.setTotal(reply.get().durableVersion);
-						myQueueInfo->value.smoothLatestVersion.setTotal(reply.get().version);
-					}
-
-					myQueueInfo->value.busiestReadTag = reply.get().busiestTag;
-					myQueueInfo->value.busiestReadTagFractionalBusyness = reply.get().busiestTagFractionalBusyness;
-					myQueueInfo->value.busiestReadTagRate = reply.get().busiestTagRate;
-				} else {
-					if (myQueueInfo->value.valid) {
-						TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id());
-					}
-					myQueueInfo->value.valid = false;
-				}
-
-				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-				     IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
-				                                                    FailureStatus(false)));
-			}
-		} catch (...) {
-			// including cancellation
-			self->storageQueueInfo.erase(myQueueInfo);
-			throw;
-		}
-	}
-
-	ACTOR static Future<Void> trackTLogQueueInfo(RatekeeperData* self, TLogInterface tli) {
-		self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
-		state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
-		TraceEvent("RkTracking", self->id).detail("TransactionLog", tli.id());
-		try {
-			loop {
-				ErrorOr<TLogQueuingMetricsReply> reply = wait(tli.getQueuingMetrics.getReplyUnlessFailedFor(
-				    TLogQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-				if (reply.present()) {
-					myQueueInfo->value.valid = true;
-					myQueueInfo->value.prevReply = myQueueInfo->value.lastReply;
-					myQueueInfo->value.lastReply = reply.get();
-					if (myQueueInfo->value.prevReply.instanceID != reply.get().instanceID) {
-						myQueueInfo->value.smoothDurableBytes.reset(reply.get().bytesDurable);
-						myQueueInfo->value.verySmoothDurableBytes.reset(reply.get().bytesDurable);
-						myQueueInfo->value.smoothInputBytes.reset(reply.get().bytesInput);
-						myQueueInfo->value.smoothFreeSpace.reset(reply.get().storageBytes.available);
-						myQueueInfo->value.smoothTotalSpace.reset(reply.get().storageBytes.total);
-					} else {
-						self->smoothTotalDurableBytes.addDelta(reply.get().bytesDurable -
-						                                       myQueueInfo->value.prevReply.bytesDurable);
-						myQueueInfo->value.smoothDurableBytes.setTotal(reply.get().bytesDurable);
-						myQueueInfo->value.verySmoothDurableBytes.setTotal(reply.get().bytesDurable);
-						myQueueInfo->value.smoothInputBytes.setTotal(reply.get().bytesInput);
-						myQueueInfo->value.smoothFreeSpace.setTotal(reply.get().storageBytes.available);
-						myQueueInfo->value.smoothTotalSpace.setTotal(reply.get().storageBytes.total);
-					}
-				} else {
-					if (myQueueInfo->value.valid) {
-						TraceEvent("RkTLogDidNotRespond", self->id).detail("TransactionLog", tli.id());
-					}
-					myQueueInfo->value.valid = false;
-				}
-
-				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-				     IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
-				                                                    FailureStatus(false)));
-			}
-		} catch (...) {
-			// including cancellation
-			self->tlogQueueInfo.erase(myQueueInfo);
-			throw;
-		}
-	}
-
-	ACTOR static Future<Void> trackEachStorageServer(
-	    RatekeeperData* self,
-	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-		state Map<UID, Future<Void>> actors;
-		state Promise<Void> err;
-		loop choose {
-			when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
-				wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
-				if (change.second.present()) {
-					if (!change.second.get().isTss()) {
-						auto& a = actors[change.first];
-						a = Future<Void>();
-						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
-					}
-				} else
-					actors.erase(change.first);
-			}
-			when(wait(err.getFuture())) {}
-		}
-	}
-
-	ACTOR static Future<Void> monitorThrottlingChanges(RatekeeperData* self) {
-		state bool committed = false;
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
-
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-
-					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
-					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
-
-					if (!committed) {
-						BinaryWriter limitWriter(Unversioned());
-						limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
-						tr.set(tagThrottleLimitKey, limitWriter.toValue());
-					}
-
-					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
-
-					if (autoThrottlingEnabled.get().present() &&
-					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
-						TEST(true); // Auto-throttling disabled
-						if (self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
-						}
-						self->autoThrottlingEnabled = false;
-					} else if (autoThrottlingEnabled.get().present() &&
-					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
-						TEST(true); // Auto-throttling enabled
-						if (!self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
-						}
-						self->autoThrottlingEnabled = true;
-					} else {
-						TEST(true); // Auto-throttling unspecified
-						if (autoThrottlingEnabled.get().present()) {
-							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
-							    .detail("Value", autoThrottlingEnabled.get().get());
-						}
-						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
-						if (!committed)
-							tr.set(tagThrottleAutoEnabledKey,
-							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
-					}
-
-					RkTagThrottleCollection updatedTagThrottles;
-
-					TraceEvent("RatekeeperReadThrottledTags", self->id)
-					    .detail("NumThrottledTags", throttledTagKeys.get().size());
-					for (auto entry : throttledTagKeys.get()) {
-						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
-						TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
-
-						ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
-
-						if (tagValue.expirationTime == 0 ||
-						    tagValue.expirationTime > now() + tagValue.initialDuration) {
-							TEST(true); // Converting tag throttle duration to absolute time
-							tagValue.expirationTime = now() + tagValue.initialDuration;
-							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
-							wr << tagValue;
-							state Value value = wr.toValue();
-
-							tr.set(entry.key, value);
-						}
-
-						if (tagValue.expirationTime > now()) {
-							TransactionTag tag = *tagKey.tags.begin();
-							Optional<ClientTagThrottleLimits> oldLimits =
-							    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
-
-							if (tagKey.throttleType == TagThrottleType::AUTO) {
-								updatedTagThrottles.autoThrottleTag(
-								    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
-								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
-									updatedTagThrottles.busyReadTagCount++;
-								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
-									updatedTagThrottles.busyWriteTagCount++;
-								}
-							} else {
-								updatedTagThrottles.manualThrottleTag(self->id,
-								                                      tag,
-								                                      tagKey.priority,
-								                                      tagValue.tpsRate,
-								                                      tagValue.expirationTime,
-								                                      oldLimits);
-							}
-						}
-					}
-
-					self->throttledTags = std::move(updatedTagThrottles);
-					++self->throttledTagChangeId;
-
-					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
-					wait(tr.commit());
-					committed = true;
-
-					wait(watchFuture);
-					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
-					TEST(true); // Tag throttle changes detected
-					break;
-				} catch (Error& e) {
-					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
-					wait(tr.onError(e));
-				}
-			}
-		}
-	}
-
-	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-		state RatekeeperData self(rkInterf.id(),
-		                          openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
-		state Future<Void> timeout = Void();
-		state std::vector<Future<Void>> tlogTrackers;
-		state std::vector<TLogInterface> tlogInterfs;
-		state Promise<Void> err;
-		state Future<Void> collection = actorCollection(self.addActor.getFuture());
-
-		TraceEvent("RatekeeperStarting", rkInterf.id());
-		self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
-		self.addActor.send(self.configurationMonitor());
-
-		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
-		self.addActor.send(self.monitorServerListChange(serverChanges));
-		self.addActor.send(self.trackEachStorageServer(serverChanges.getFuture()));
-		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
-
-		self.addActor.send(self.monitorThrottlingChanges());
-		RatekeeperData* selfPtr = &self; // let flow compiler capture self
-		self.addActor.send(recurring([selfPtr]() { selfPtr->refreshStorageServerCommitCost(); },
-		                             SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
-
-		TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
-		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
-		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
-		    .detail(
-		        "Rate",
-		        (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
-		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-		             2.0));
-
-		TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
-		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
-		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
-		    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
-		    .detail(
-		        "Rate",
-		        (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
-		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-		             2.0));
-
-		tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-		tlogTrackers.reserve(tlogInterfs.size());
-		for (int i = 0; i < tlogInterfs.size(); i++) {
-			tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
-		}
-
-		self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
-
-		try {
-			state bool lastLimited = false;
-			loop choose {
-				when(wait(timeout)) {
-					self.updateRate(&self.normalLimits);
-					self.updateRate(&self.batchLimits);
-
-					lastLimited = self.smoothReleasedTransactions.smoothRate() >
-					              SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
-					double tooOld = now() - 1.0;
-					for (auto p = self.grvProxyInfo.begin(); p != self.grvProxyInfo.end();) {
-						if (p->second.lastUpdateTime < tooOld)
-							p = self.grvProxyInfo.erase(p);
-						else
-							++p;
-					}
-					timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
-				}
-				when(GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
-					GetRateInfoReply reply;
-
-					auto& p = self.grvProxyInfo[req.requesterID];
-					//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
-					if (p.totalTransactions > 0) {
-						self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
-
-						for (auto tag : req.throttledTagCounts) {
-							self.throttledTags.addRequests(tag.first, tag.second);
-						}
-					}
-					if (p.batchTransactions > 0) {
-						self.smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions -
-						                                              p.batchTransactions);
-					}
-
-					p.totalTransactions = req.totalReleasedTransactions;
-					p.batchTransactions = req.batchReleasedTransactions;
-					p.lastUpdateTime = now();
-
-					reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
-					reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
-					reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
-
-					if (p.lastThrottledTagChangeId != self.throttledTagChangeId ||
-					    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
-						p.lastThrottledTagChangeId = self.throttledTagChangeId;
-						p.lastTagPushTime = now();
-
-						reply.throttledTags = self.throttledTags.getClientRates(self.autoThrottlingEnabled);
-						bool returningTagsToProxy =
-						    reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
-						TEST(returningTagsToProxy); // Returning tag throttles to a proxy
-					}
-
-					reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
-					reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
-					reply.healthMetrics.batchLimited = lastLimited;
-
-					req.reply.send(reply);
-				}
-				when(HaltRatekeeperRequest req = waitNext(rkInterf.haltRatekeeper.getFuture())) {
-					req.reply.send(Void());
-					TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
-					break;
-				}
-				when(ReportCommitCostEstimationRequest req =
-				         waitNext(rkInterf.reportCommitCostEstimation.getFuture())) {
-					self.updateCommitCostEstimation(req.ssTrTagCommitCost);
-					req.reply.send(Void());
-				}
-				when(wait(err.getFuture())) {}
-				when(wait(dbInfo->onChange())) {
-					if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
-						tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-						tlogTrackers = std::vector<Future<Void>>();
-						for (int i = 0; i < tlogInterfs.size(); i++)
-							tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
-					}
-					self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
-				}
-				when(wait(collection)) {
-					ASSERT(false);
-					throw internal_error();
-				}
-			}
-		} catch (Error& err) {
-			TraceEvent("RatekeeperDied", rkInterf.id()).error(err, true);
-		}
-		return Void();
-	}
-
-}; // class RatekeeperDataImpl
-
-Future<Void> RatekeeperData::configurationMonitor() {
-	return RatekeeperDataImpl::configurationMonitor(this);
-}
-
-Future<Void> RatekeeperData::monitorServerListChange(
-    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperDataImpl::monitorServerListChange(this, serverChanges);
-}
-
-Future<Void> RatekeeperData::trackEachStorageServer(
-    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperDataImpl::trackEachStorageServer(this, serverChanges);
-}
-
-Future<Void> RatekeeperData::trackStorageServerQueueInfo(StorageServerInterface ssi) {
-	return RatekeeperDataImpl::trackStorageServerQueueInfo(this, ssi);
-}
-
-Future<Void> RatekeeperData::trackTLogQueueInfo(TLogInterface tli) {
-	return RatekeeperDataImpl::trackTLogQueueInfo(this, tli);
-}
-
-Future<Void> RatekeeperData::monitorThrottlingChanges() {
-	return RatekeeperDataImpl::monitorThrottlingChanges(this);
-}
-
-Future<Void> RatekeeperData::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	return RatekeeperDataImpl::run(rkInterf, dbInfo);
-}
-
-RatekeeperData::RatekeeperData(UID id, Database db)
-  : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-    actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
-    lastBusiestCommitTagPick(0), throttledTagChangeId(0), normalLimits(TransactionPriority::DEFAULT,
-                                                                       "",
-                                                                       SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
-                                                                       SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
-                                                                       SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
-                                                                       SERVER_KNOBS->SPRING_BYTES_TLOG,
-                                                                       SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
-                                                                       SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
-    batchLimits(TransactionPriority::BATCH,
-                "Batch",
-                SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
-                SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH,
-                SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
-                SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
-                SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
-    autoThrottlingEnabled(false) {
-	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
-	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
-}
-
-void RatekeeperData::updateCommitCostEstimation(
-    UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation) {
-	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
-		auto tagCostIt = costEstimation.find(it->key);
-		if (tagCostIt == costEstimation.end())
-			continue;
-		for (const auto& [tagName, cost] : tagCostIt->second) {
-			it->value.tagCostEst[tagName] += cost;
-			it->value.totalWriteCosts += cost.getCostSum();
-			it->value.totalWriteOps += cost.getOpsSum();
-		}
-	}
-}
-
-void RatekeeperData::updateRate(RatekeeperLimits* limits) {
-	// double controlFactor = ;  // dt / eFoldingTime
-
-	double actualTps = smoothReleasedTransactions.smoothRate();
-	actualTpsMetric = (int64_t)actualTps;
-	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this
-	// value
-	actualTps =
-	    std::max(std::max(1.0, actualTps), smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
-
-	if (actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
-		actualTpsHistory.pop_front();
-	}
-	actualTpsHistory.push_back(actualTps);
-
-	limits->tpsLimit = std::numeric_limits<double>::infinity();
-	UID reasonID = UID();
-	limitReason_t limitReason = limitReason_t::unlimited;
-
-	int sscount = 0;
-
-	int64_t worstFreeSpaceStorageServer = std::numeric_limits<int64_t>::max();
-	int64_t worstStorageQueueStorageServer = 0;
-	int64_t limitingStorageQueueStorageServer = 0;
-	int64_t worstDurabilityLag = 0;
-
-	std::multimap<double, StorageQueueInfo*> storageTpsLimitReverseIndex;
-	std::multimap<int64_t, StorageQueueInfo*> storageDurabilityLagReverseIndex;
-
-	std::map<UID, limitReason_t> ssReasons;
-
-	bool printRateKeepLimitReasonDetails =
-	    SERVER_KNOBS->RATEKEEPER_PRINT_LIMIT_REASON &&
-	    (deterministicRandom()->random01() < SERVER_KNOBS->RATEKEEPER_LIMIT_REASON_SAMPLE_RATE);
-
-	// Look at each storage server's write queue and local rate, compute and store the desired rate ratio
-	for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) {
-		auto& ss = i->value;
-		if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
-			continue;
-		++sscount;
-
-		limitReason_t ssLimitReason = limitReason_t::unlimited;
-
-		int64_t minFreeSpace =
-		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
-		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * ss.smoothTotalSpace.smoothTotal()));
-
-		worstFreeSpaceStorageServer =
-		    std::min(worstFreeSpaceStorageServer, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace);
-
-		int64_t springBytes = std::max<int64_t>(
-		    1, std::min<int64_t>(limits->storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
-		int64_t targetBytes = std::max<int64_t>(
-		    1, std::min(limits->storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
-		if (targetBytes != limits->storageTargetBytes) {
-			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
-				ssLimitReason = limitReason_t::storage_server_min_free_space;
-			} else {
-				ssLimitReason = limitReason_t::storage_server_min_free_space_ratio;
-			}
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("Reason", ssLimitReason)
-				    .detail("SSID", ss.id)
-				    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
-				    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
-				    .detail("TargetBytes", targetBytes)
-				    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
-				    .detail("MinFreeSpace", minFreeSpace);
-			}
-		}
-
-		int64_t storageQueue = ss.lastReply.bytesInput - ss.smoothDurableBytes.smoothTotal();
-		worstStorageQueueStorageServer = std::max(worstStorageQueueStorageServer, storageQueue);
-
-		int64_t storageDurabilityLag = ss.smoothLatestVersion.smoothTotal() - ss.smoothDurableVersion.smoothTotal();
-		worstDurabilityLag = std::max(worstDurabilityLag, storageDurabilityLag);
-
-		storageDurabilityLagReverseIndex.insert(std::make_pair(-1 * storageDurabilityLag, &ss));
-
-		auto& ssMetrics = healthMetrics.storageStats[ss.id];
-		ssMetrics.storageQueue = storageQueue;
-		ssMetrics.storageDurabilityLag = storageDurabilityLag;
-		ssMetrics.cpuUsage = ss.lastReply.cpuUsage;
-		ssMetrics.diskUsage = ss.lastReply.diskUsage;
-
-		double targetRateRatio = std::min((storageQueue - targetBytes + springBytes) / (double)springBytes, 2.0);
-
-		if (limits->priority == TransactionPriority::DEFAULT) {
-			tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag);
-		}
-
-		double inputRate = ss.smoothInputBytes.smoothRate();
-		// inputRate = std::max( inputRate, actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE );
-
-		/*if( deterministicRandom()->random01() < 0.1 ) {
-		  std::string name = "RatekeeperUpdateRate" + limits.context;
-		  TraceEvent(name, ss.id)
-		  .detail("MinFreeSpace", minFreeSpace)
-		  .detail("SpringBytes", springBytes)
-		  .detail("TargetBytes", targetBytes)
-		  .detail("SmoothTotalSpaceTotal", ss.smoothTotalSpace.smoothTotal())
-		  .detail("SmoothFreeSpaceTotal", ss.smoothFreeSpace.smoothTotal())
-		  .detail("LastReplyBytesInput", ss.lastReply.bytesInput)
-		  .detail("SmoothDurableBytesTotal", ss.smoothDurableBytes.smoothTotal())
-		  .detail("TargetRateRatio", targetRateRatio)
-		  .detail("SmoothInputBytesRate", ss.smoothInputBytes.smoothRate())
-		  .detail("ActualTPS", actualTps)
-		  .detail("InputRate", inputRate)
-		  .detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate())
-		  .detail("B", b);
-		  }*/
-
-		// Don't let any storage server use up its target bytes faster than its MVCC window!
-		double maxBytesPerSecond =
-		    (targetBytes - springBytes) /
-		    ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0);
-		double limitTps = std::min(actualTps * maxBytesPerSecond / std::max(1.0e-8, inputRate),
-		                           maxBytesPerSecond * SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-		if (ssLimitReason == limitReason_t::unlimited)
-			ssLimitReason = limitReason_t::storage_server_write_bandwidth_mvcc;
-
-		if (targetRateRatio > 0 && inputRate > 0) {
-			ASSERT(inputRate != 0);
-			double smoothedRate =
-			    std::max(ss.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-			double x = smoothedRate / (inputRate * targetRateRatio);
-			double lim = actualTps * x;
-			if (lim < limitTps) {
-				double oldLimitTps = limitTps;
-				limitTps = lim;
-				if (ssLimitReason == limitReason_t::unlimited ||
-				    ssLimitReason == limitReason_t::storage_server_write_bandwidth_mvcc) {
-					if (printRateKeepLimitReasonDetails) {
-						TraceEvent("RatekeeperLimitReasonDetails")
-						    .detail("Reason", limitReason_t::storage_server_write_queue_size)
-						    .detail("FromReason", ssLimitReason)
-						    .detail("SSID", ss.id)
-						    .detail("SSSmoothTotalSpace", ss.smoothTotalSpace.smoothTotal())
-						    .detail("LimitsStorageTargetBytes", limits->storageTargetBytes)
-						    .detail("LimitsStorageSpringBytes", limits->storageSpringBytes)
-						    .detail("SSSmoothFreeSpace", ss.smoothFreeSpace.smoothTotal())
-						    .detail("MinFreeSpace", minFreeSpace)
-						    .detail("SSLastReplyBytesInput", ss.lastReply.bytesInput)
-						    .detail("SSSmoothDurableBytes", ss.smoothDurableBytes.smoothTotal())
-						    .detail("StorageQueue", storageQueue)
-						    .detail("TargetBytes", targetBytes)
-						    .detail("SpringBytes", springBytes)
-						    .detail("SSVerySmoothDurableBytesSmoothRate", ss.verySmoothDurableBytes.smoothRate())
-						    .detail("SmoothedRate", smoothedRate)
-						    .detail("X", x)
-						    .detail("ActualTps", actualTps)
-						    .detail("Lim", lim)
-						    .detail("LimitTps", oldLimitTps)
-						    .detail("InputRate", inputRate)
-						    .detail("TargetRateRatio", targetRateRatio);
-					}
-					ssLimitReason = limitReason_t::storage_server_write_queue_size;
-				}
-			}
-		}
-
-		storageTpsLimitReverseIndex.insert(std::make_pair(limitTps, &ss));
-
-		if (limitTps < limits->tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space ||
-		                                    ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
-			reasonID = ss.id;
-			limits->tpsLimit = limitTps;
-			limitReason = ssLimitReason;
-		}
-
-		ssReasons[ss.id] = ssLimitReason;
-	}
-
-	std::set<Optional<Standalone<StringRef>>> ignoredMachines;
-	for (auto ss = storageTpsLimitReverseIndex.begin();
-	     ss != storageTpsLimitReverseIndex.end() && ss->first < limits->tpsLimit;
-	     ++ss) {
-		if (ignoredMachines.size() <
-		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
-			ignoredMachines.insert(ss->second->locality.zoneId());
-			continue;
-		}
-		if (ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
-			continue;
-		}
-
-		limitingStorageQueueStorageServer =
-		    ss->second->lastReply.bytesInput - ss->second->smoothDurableBytes.smoothTotal();
-		limits->tpsLimit = ss->first;
-		reasonID = storageTpsLimitReverseIndex.begin()->second->id; // Although we aren't controlling based on the worst
-		// SS, we still report it as the limiting process
-		limitReason = ssReasons[reasonID];
-		break;
-	}
-
-	// Calculate limited durability lag
-	int64_t limitingDurabilityLag = 0;
-
-	std::set<Optional<Standalone<StringRef>>> ignoredDurabilityLagMachines;
-	for (auto ss = storageDurabilityLagReverseIndex.begin(); ss != storageDurabilityLagReverseIndex.end(); ++ss) {
-		if (ignoredDurabilityLagMachines.size() <
-		    std::min(configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
-			ignoredDurabilityLagMachines.insert(ss->second->locality.zoneId());
-			continue;
-		}
-		if (ignoredDurabilityLagMachines.count(ss->second->locality.zoneId()) > 0) {
-			continue;
-		}
-
-		limitingDurabilityLag = -1 * ss->first;
-		if (limitingDurabilityLag > limits->durabilityLagTargetVersions &&
-		    actualTpsHistory.size() > SERVER_KNOBS->NEEDED_TPS_HISTORY_SAMPLES) {
-			if (limits->durabilityLagLimit == std::numeric_limits<double>::infinity()) {
-				double maxTps = 0;
-				for (int i = 0; i < actualTpsHistory.size(); i++) {
-					maxTps = std::max(maxTps, actualTpsHistory[i]);
-				}
-				limits->durabilityLagLimit = SERVER_KNOBS->INITIAL_DURABILITY_LAG_MULTIPLIER * maxTps;
-			}
-			if (limitingDurabilityLag > limits->lastDurabilityLag) {
-				limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_REDUCTION_RATE * limits->durabilityLagLimit;
-			}
-			if (limits->durabilityLagLimit < limits->tpsLimit) {
-				if (printRateKeepLimitReasonDetails) {
-					TraceEvent("RatekeeperLimitReasonDetails")
-					    .detail("SSID", ss->second->id)
-					    .detail("Reason", limitReason_t::storage_server_durability_lag)
-					    .detail("LimitsDurabilityLagLimit", limits->durabilityLagLimit)
-					    .detail("LimitsTpsLimit", limits->tpsLimit)
-					    .detail("LimitingDurabilityLag", limitingDurabilityLag)
-					    .detail("LimitsLastDurabilityLag", limits->lastDurabilityLag);
-				}
-				limits->tpsLimit = limits->durabilityLagLimit;
-				limitReason = limitReason_t::storage_server_durability_lag;
-			}
-		} else if (limits->durabilityLagLimit != std::numeric_limits<double>::infinity() &&
-		           limitingDurabilityLag >
-		               limits->durabilityLagTargetVersions - SERVER_KNOBS->DURABILITY_LAG_UNLIMITED_THRESHOLD) {
-			limits->durabilityLagLimit = SERVER_KNOBS->DURABILITY_LAG_INCREASE_RATE * limits->durabilityLagLimit;
-		} else {
-			limits->durabilityLagLimit = std::numeric_limits<double>::infinity();
-		}
-		limits->lastDurabilityLag = limitingDurabilityLag;
-		break;
-	}
-
-	healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
-	healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
-	healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
-	healthMetrics.limitingStorageDurabilityLag = limitingDurabilityLag;
-
-	double writeToReadLatencyLimit = 0;
-	Version worstVersionLag = 0;
-	Version limitingVersionLag = 0;
-
-	{
-		Version minSSVer = std::numeric_limits<Version>::max();
-		Version minLimitingSSVer = std::numeric_limits<Version>::max();
-		for (const auto& it : storageQueueInfo) {
-			auto& ss = it.value;
-			if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC))
-				continue;
-
-			minSSVer = std::min(minSSVer, ss.lastReply.version);
-
-			// Machines that ratekeeper isn't controlling can fall arbitrarily far behind
-			if (ignoredMachines.count(it.value.locality.zoneId()) == 0) {
-				minLimitingSSVer = std::min(minLimitingSSVer, ss.lastReply.version);
-			}
-		}
-
-		Version maxTLVer = std::numeric_limits<Version>::min();
-		for (const auto& it : tlogQueueInfo) {
-			auto& tl = it.value;
-			if (!tl.valid)
-				continue;
-			maxTLVer = std::max(maxTLVer, tl.lastReply.v);
-		}
-
-		if (minSSVer != std::numeric_limits<Version>::max() && maxTLVer != std::numeric_limits<Version>::min()) {
-			// writeToReadLatencyLimit: 0 = infinte speed; 1 = TL durable speed ; 2 = half TL durable speed
-			writeToReadLatencyLimit =
-			    ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference / 2) / (limits->maxVersionDifference / 4);
-			worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
-			limitingVersionLag = std::max((Version)0, maxTLVer - minLimitingSSVer);
-		}
-	}
-
-	int64_t worstFreeSpaceTLog = std::numeric_limits<int64_t>::max();
-	int64_t worstStorageQueueTLog = 0;
-	int tlcount = 0;
-	for (auto& it : tlogQueueInfo) {
-		auto& tl = it.value;
-		if (!tl.valid)
-			continue;
-		++tlcount;
-
-		limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue;
-
-		int64_t minFreeSpace =
-		    std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
-		             (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.smoothTotalSpace.smoothTotal()));
-
-		worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace);
-
-		int64_t springBytes = std::max<int64_t>(
-		    1, std::min<int64_t>(limits->logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
-		int64_t targetBytes = std::max<int64_t>(
-		    1, std::min(limits->logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
-		if (targetBytes != limits->logTargetBytes) {
-			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
-				tlogLimitReason = limitReason_t::log_server_min_free_space;
-			} else {
-				tlogLimitReason = limitReason_t::log_server_min_free_space_ratio;
-			}
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("TLogID", tl.id)
-				    .detail("Reason", tlogLimitReason)
-				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-				    .detail("TargetBytes", targetBytes)
-				    .detail("MinFreeSpace", minFreeSpace);
-			}
-		}
-
-		int64_t queue = tl.lastReply.bytesInput - tl.smoothDurableBytes.smoothTotal();
-		healthMetrics.tLogQueue[tl.id] = queue;
-		int64_t b = queue - targetBytes;
-		worstStorageQueueTLog = std::max(worstStorageQueueTLog, queue);
-
-		if (tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2) {
-			if (now() - lastWarning > 5.0) {
-				lastWarning = now();
-				TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero", id).detail("ReasonId", tl.id);
-			}
-			reasonID = tl.id;
-			limitReason = limitReason_t::log_server_min_free_space;
-			limits->tpsLimit = 0.0;
-		}
-
-		double targetRateRatio = std::min((b + springBytes) / (double)springBytes, 2.0);
-
-		if (writeToReadLatencyLimit > targetRateRatio) {
-			if (printRateKeepLimitReasonDetails) {
-				TraceEvent("RatekeeperLimitReasonDetails")
-				    .detail("TLogID", tl.id)
-				    .detail("Reason", limitReason_t::storage_server_readable_behind)
-				    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-				    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-				    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
-				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-				    .detail("SpringBytes", springBytes)
-				    .detail("TargetBytes", targetBytes)
-				    .detail("TLLastReplyBytesInput", tl.lastReply.bytesInput)
-				    .detail("TLSmoothDurableBytes", tl.smoothDurableBytes.smoothTotal())
-				    .detail("Queue", queue)
-				    .detail("B", b)
-				    .detail("TargetRateRatio", targetRateRatio)
-				    .detail("WriteToReadLatencyLimit", writeToReadLatencyLimit)
-				    .detail("MinFreeSpace", minFreeSpace)
-				    .detail("LimitsMaxVersionDifference", limits->maxVersionDifference);
-			}
-			targetRateRatio = writeToReadLatencyLimit;
-			tlogLimitReason = limitReason_t::storage_server_readable_behind;
-		}
-
-		double inputRate = tl.smoothInputBytes.smoothRate();
-
-		if (targetRateRatio > 0) {
-			double smoothedRate =
-			    std::max(tl.verySmoothDurableBytes.smoothRate(), actualTps / SERVER_KNOBS->MAX_TRANSACTIONS_PER_BYTE);
-			double x = smoothedRate / (inputRate * targetRateRatio);
-			if (targetRateRatio < .75) //< FIXME: KNOB for 2.0
-				x = std::max(x, 0.95);
-			double lim = actualTps * x;
-			if (lim < limits->tpsLimit) {
-				limits->tpsLimit = lim;
-				reasonID = tl.id;
-				limitReason = tlogLimitReason;
-			}
-		}
-		if (inputRate > 0) {
-			// Don't let any tlogs use up its target bytes faster than its MVCC window!
-			double x =
-			    ((targetBytes - springBytes) /
-			     ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-			      2.0)) /
-			    inputRate;
-			double lim = actualTps * x;
-			if (lim < limits->tpsLimit) {
-				if (printRateKeepLimitReasonDetails) {
-					TraceEvent("RatekeeperLimitReasonDetails")
-					    .detail("Reason", limitReason_t::log_server_mvcc_write_bandwidth)
-					    .detail("TLogID", tl.id)
-					    .detail("MinFreeSpace", minFreeSpace)
-					    .detail("TLSmoothFreeSpace", tl.smoothFreeSpace.smoothTotal())
-					    .detail("TLSmoothTotalSpace", tl.smoothTotalSpace.smoothTotal())
-					    .detail("LimitsLogSpringBytes", limits->logSpringBytes)
-					    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
-					    .detail("SpringBytes", springBytes)
-					    .detail("TargetBytes", targetBytes)
-					    .detail("InputRate", inputRate)
-					    .detail("X", x)
-					    .detail("ActualTps", actualTps)
-					    .detail("Lim", lim)
-					    .detail("LimitsTpsLimit", limits->tpsLimit);
-				}
-				limits->tpsLimit = lim;
-				reasonID = tl.id;
-				limitReason = limitReason_t::log_server_mvcc_write_bandwidth;
-			}
-		}
-	}
-
-	healthMetrics.worstTLogQueue = worstStorageQueueTLog;
-
-	limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
-
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
-		limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
-	}
-
-	int64_t totalDiskUsageBytes = 0;
-	for (auto& t : tlogQueueInfo) {
-		if (t.value.valid) {
-			totalDiskUsageBytes += t.value.lastReply.storageBytes.used;
-		}
-	}
-	for (auto& s : storageQueueInfo) {
-		if (s.value.valid) {
-			totalDiskUsageBytes += s.value.lastReply.storageBytes.used;
-		}
-	}
-
-	if (now() - lastSSListFetchedTimestamp > SERVER_KNOBS->STORAGE_SERVER_LIST_FETCH_TIMEOUT) {
-		limits->tpsLimit = 0.0;
-		limitReason = limitReason_t::storage_server_list_fetch_failed;
-		reasonID = UID();
-		TraceEvent(SevWarnAlways, "RkSSListFetchTimeout", id).suppressFor(1.0);
-	} else if (limits->tpsLimit == std::numeric_limits<double>::infinity()) {
-		limits->tpsLimit = SERVER_KNOBS->RATEKEEPER_DEFAULT_LIMIT;
-	}
-
-	limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
-	limits->reasonMetric = limitReason;
-
-	if (deterministicRandom()->random01() < 0.1) {
-		const std::string& name = limits->rkUpdateEventCacheHolder.getPtr()->trackingKey;
-		TraceEvent(name.c_str(), id)
-		    .detail("TPSLimit", limits->tpsLimit)
-		    .detail("Reason", limitReason)
-		    .detail("ReasonServerID", reasonID == UID() ? std::string() : Traceable<UID>::toString(reasonID))
-		    .detail("ReleasedTPS", smoothReleasedTransactions.smoothRate())
-		    .detail("ReleasedBatchTPS", smoothBatchReleasedTransactions.smoothRate())
-		    .detail("TPSBasis", actualTps)
-		    .detail("StorageServers", sscount)
-		    .detail("GrvProxies", grvProxyInfo.size())
-		    .detail("TLogs", tlcount)
-		    .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer)
-		    .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog)
-		    .detail("WorstStorageServerQueue", worstStorageQueueStorageServer)
-		    .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer)
-		    .detail("WorstTLogQueue", worstStorageQueueTLog)
-		    .detail("TotalDiskUsageBytes", totalDiskUsageBytes)
-		    .detail("WorstStorageServerVersionLag", worstVersionLag)
-		    .detail("LimitingStorageServerVersionLag", limitingVersionLag)
-		    .detail("WorstStorageServerDurabilityLag", worstDurabilityLag)
-		    .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag)
-		    .detail("TagsAutoThrottled", throttledTags.autoThrottleCount())
-		    .detail("TagsAutoThrottledBusyRead", throttledTags.busyReadTagCount)
-		    .detail("TagsAutoThrottledBusyWrite", throttledTags.busyWriteTagCount)
-		    .detail("TagsManuallyThrottled", throttledTags.manualThrottleCount())
-		    .detail("AutoThrottlingEnabled", autoThrottlingEnabled)
-		    .trackLatest(name);
-	}
-}
-
-Future<Void> RatekeeperData::refreshStorageServerCommitCost() {
-	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
-		lastBusiestCommitTagPick = now();
-		return Void();
-	}
-	double elapsed = now() - lastBusiestCommitTagPick;
-	// for each SS, select the busiest commit tag from ssTrTagCommitCost
-	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
-		it->value.busiestWriteTag.reset();
-		TransactionTag busiestTag;
-		TransactionCommitCostEstimation maxCost;
-		double maxRate = 0, maxBusyness = 0;
-		for (const auto& [tag, cost] : it->value.tagCostEst) {
-			double rate = cost.getCostSum() / elapsed;
-			if (rate > maxRate) {
-				busiestTag = tag;
-				maxRate = rate;
-				maxCost = cost;
-			}
-		}
-		if (maxRate > SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE) {
-			it->value.busiestWriteTag = busiestTag;
-			// TraceEvent("RefreshSSCommitCost").detail("TotalWriteCost", it->value.totalWriteCost).detail("TotalWriteOps",it->value.totalWriteOps);
-			ASSERT(it->value.totalWriteCosts > 0);
-			maxBusyness = double(maxCost.getCostSum()) / it->value.totalWriteCosts;
-			it->value.busiestWriteTagFractionalBusyness = maxBusyness;
-			it->value.busiestWriteTagRate = maxRate;
-		}
-
-		TraceEvent("BusiestWriteTag", it->key)
-		    .detail("Elapsed", elapsed)
-		    .detail("Tag", printable(busiestTag))
-		    .detail("TagOps", maxCost.getOpsSum())
-		    .detail("TagCost", maxCost.getCostSum())
-		    .detail("TotalCost", it->value.totalWriteCosts)
-		    .detail("Reported", it->value.busiestWriteTag.present())
-		    .trackLatest(it->value.busiestWriteTagEventHolder->trackingKey);
-
-		// reset statistics
-		it->value.tagCostEst.clear();
-		it->value.totalWriteOps = 0;
-		it->value.totalWriteCosts = 0;
-	}
-	lastBusiestCommitTagPick = now();
-	return Void();
-}
-
-void RatekeeperData::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
-	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
-	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
-	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
-		TEST(true); // Transaction tag auto-throttled
-		Optional<double> clientRate = throttledTags.autoThrottleTag(id, tag, busyness);
-		if (clientRate.present()) {
-			TagSet tags;
-			tags.addTag(tag);
-
-			Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(db.getPtr());
-			addActor.send(ThrottleApi::throttleTags(dbRef,
-			                                        tags,
-			                                        clientRate.get(),
-			                                        SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                        TagThrottleType::AUTO,
-			                                        TransactionPriority::DEFAULT,
-			                                        now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                        reason));
-		}
-	}
-}
-
-void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
-	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
-	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
-	// future
-	if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
-	    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
-		if (ss.busiestWriteTag.present()) {
-			tryAutoThrottleTag(ss.busiestWriteTag.get(),
-			                   ss.busiestWriteTagRate,
-			                   ss.busiestWriteTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_WRITE);
-		}
-		if (ss.busiestReadTag.present()) {
-			tryAutoThrottleTag(ss.busiestReadTag.get(),
-			                   ss.busiestReadTagRate,
-			                   ss.busiestReadTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_READ);
-		}
-	}
-}
-
-ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	wait(RatekeeperData::run(rkInterf, dbInfo));
-	return Void();
-}

From d6c5239080d9d840dda753b5c145ee6849ab85af Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 12:36:44 -0800
Subject: [PATCH 062/210] Fix copyright headers

---
 fdbserver/Ratekeeper.actor.cpp | 2 +-
 fdbserver/Ratekeeper.h         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 38564b3359..34e66e0fb9 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -1,5 +1,5 @@
 /*
- * RatekeeperData.actor.cpp
+ * Ratekeeper.actor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index 4340796a2e..a0857a4f04 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -1,5 +1,5 @@
 /*
- * RatekeeperData.h
+ * Ratekeeper.h
  *
  * This source file is part of the FoundationDB open source project
  *

From 807463053013863e5105e7a3efaa5c2200c20b08 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 12:42:25 -0800
Subject: [PATCH 063/210] Rename class RatekeeperData to Ratekeeper

---
 fdbserver/Ratekeeper.actor.cpp | 63 +++++++++++++++++-----------------
 fdbserver/Ratekeeper.h         |  6 ++--
 2 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 34e66e0fb9..669b9d08a0 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -65,9 +65,9 @@ ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
 	}
 }
 
-class RatekeeperDataImpl {
+class RatekeeperImpl {
 public:
-	ACTOR static Future<Void> configurationMonitor(RatekeeperData* self) {
+	ACTOR static Future<Void> configurationMonitor(Ratekeeper* self) {
 		loop {
 			state ReadYourWritesTransaction tr(self->db);
 
@@ -95,7 +95,7 @@ public:
 	}
 
 	ACTOR static Future<Void> monitorServerListChange(
-	    RatekeeperData* self,
+	    Ratekeeper* self,
 	    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
 		state std::map<UID, StorageServerInterface> oldServers;
 		state Transaction tr(self->db);
@@ -140,7 +140,7 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> trackStorageServerQueueInfo(RatekeeperData* self, StorageServerInterface ssi) {
+	ACTOR static Future<Void> trackStorageServerQueueInfo(Ratekeeper* self, StorageServerInterface ssi) {
 		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
 		state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
 		TraceEvent("RkTracking", self->id)
@@ -195,7 +195,7 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> trackTLogQueueInfo(RatekeeperData* self, TLogInterface tli) {
+	ACTOR static Future<Void> trackTLogQueueInfo(Ratekeeper* self, TLogInterface tli) {
 		self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
 		state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
 		TraceEvent("RkTracking", self->id).detail("TransactionLog", tli.id());
@@ -241,7 +241,7 @@ public:
 	}
 
 	ACTOR static Future<Void> trackEachStorageServer(
-	    RatekeeperData* self,
+	    Ratekeeper* self,
 	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
 		state Map<UID, Future<Void>> actors;
 		state Promise<Void> err;
@@ -261,7 +261,7 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> monitorThrottlingChanges(RatekeeperData* self) {
+	ACTOR static Future<Void> monitorThrottlingChanges(Ratekeeper* self) {
 		state bool committed = false;
 		loop {
 			state ReadYourWritesTransaction tr(self->db);
@@ -373,8 +373,7 @@ public:
 	}
 
 	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-		state RatekeeperData self(rkInterf.id(),
-		                          openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
+		state Ratekeeper self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
 		state Future<Void> timeout = Void();
 		state std::vector<Future<Void>> tlogTrackers;
 		state std::vector<TLogInterface> tlogInterfs;
@@ -391,7 +390,7 @@ public:
 		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
 
 		self.addActor.send(self.monitorThrottlingChanges());
-		RatekeeperData* selfPtr = &self; // let flow compiler capture self
+		Ratekeeper* selfPtr = &self; // let flow compiler capture self
 		self.addActor.send(recurring([selfPtr]() { selfPtr->refreshStorageServerCommitCost(); },
 		                             SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
 
@@ -513,39 +512,39 @@ public:
 		return Void();
 	}
 
-}; // class RatekeeperDataImpl
+}; // class RatekeeperImpl
 
-Future<Void> RatekeeperData::configurationMonitor() {
-	return RatekeeperDataImpl::configurationMonitor(this);
+Future<Void> Ratekeeper::configurationMonitor() {
+	return RatekeeperImpl::configurationMonitor(this);
 }
 
-Future<Void> RatekeeperData::monitorServerListChange(
+Future<Void> Ratekeeper::monitorServerListChange(
     PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperDataImpl::monitorServerListChange(this, serverChanges);
+	return RatekeeperImpl::monitorServerListChange(this, serverChanges);
 }
 
-Future<Void> RatekeeperData::trackEachStorageServer(
+Future<Void> Ratekeeper::trackEachStorageServer(
     FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperDataImpl::trackEachStorageServer(this, serverChanges);
+	return RatekeeperImpl::trackEachStorageServer(this, serverChanges);
 }
 
-Future<Void> RatekeeperData::trackStorageServerQueueInfo(StorageServerInterface ssi) {
-	return RatekeeperDataImpl::trackStorageServerQueueInfo(this, ssi);
+Future<Void> Ratekeeper::trackStorageServerQueueInfo(StorageServerInterface ssi) {
+	return RatekeeperImpl::trackStorageServerQueueInfo(this, ssi);
 }
 
-Future<Void> RatekeeperData::trackTLogQueueInfo(TLogInterface tli) {
-	return RatekeeperDataImpl::trackTLogQueueInfo(this, tli);
+Future<Void> Ratekeeper::trackTLogQueueInfo(TLogInterface tli) {
+	return RatekeeperImpl::trackTLogQueueInfo(this, tli);
 }
 
-Future<Void> RatekeeperData::monitorThrottlingChanges() {
-	return RatekeeperDataImpl::monitorThrottlingChanges(this);
+Future<Void> Ratekeeper::monitorThrottlingChanges() {
+	return RatekeeperImpl::monitorThrottlingChanges(this);
 }
 
-Future<Void> RatekeeperData::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	return RatekeeperDataImpl::run(rkInterf, dbInfo);
+Future<Void> Ratekeeper::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	return RatekeeperImpl::run(rkInterf, dbInfo);
 }
 
-RatekeeperData::RatekeeperData(UID id, Database db)
+Ratekeeper::Ratekeeper(UID id, Database db)
   : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
@@ -571,7 +570,7 @@ RatekeeperData::RatekeeperData(UID id, Database db)
 	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
 }
 
-void RatekeeperData::updateCommitCostEstimation(
+void Ratekeeper::updateCommitCostEstimation(
     UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation) {
 	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
 		auto tagCostIt = costEstimation.find(it->key);
@@ -585,7 +584,7 @@ void RatekeeperData::updateCommitCostEstimation(
 	}
 }
 
-void RatekeeperData::updateRate(RatekeeperLimits* limits) {
+void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 	// double controlFactor = ;  // dt / eFoldingTime
 
 	double actualTps = smoothReleasedTransactions.smoothRate();
@@ -1066,7 +1065,7 @@ void RatekeeperData::updateRate(RatekeeperLimits* limits) {
 	}
 }
 
-Future<Void> RatekeeperData::refreshStorageServerCommitCost() {
+Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
 	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
 		lastBusiestCommitTagPick = now();
 		return Void();
@@ -1113,7 +1112,7 @@ Future<Void> RatekeeperData::refreshStorageServerCommitCost() {
 	return Void();
 }
 
-void RatekeeperData::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
+void Ratekeeper::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
 	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
 	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
 	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
@@ -1136,7 +1135,7 @@ void RatekeeperData::tryAutoThrottleTag(TransactionTag tag, double rate, double
 	}
 }
 
-void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
+void Ratekeeper::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
 	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
 	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
 	// future
@@ -1158,6 +1157,6 @@ void RatekeeperData::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQue
 }
 
 ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	wait(RatekeeperData::run(rkInterf, dbInfo));
+	wait(Ratekeeper::run(rkInterf, dbInfo));
 	return Void();
 }
diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index a0857a4f04..d29dffb99b 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -502,7 +502,7 @@ struct RatekeeperLimits {
 	    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
 };
 
-struct RatekeeperData {
+struct Ratekeeper {
 	// Differentiate from GrvProxyInfo in DatabaseContext.h
 	struct GrvProxyInfo {
 		int64_t totalTransactions;
@@ -523,7 +523,7 @@ struct RatekeeperData {
 	Map<UID, StorageQueueInfo> storageQueueInfo;
 	Map<UID, TLogQueueInfo> tlogQueueInfo;
 
-	std::map<UID, RatekeeperData::GrvProxyInfo> grvProxyInfo;
+	std::map<UID, Ratekeeper::GrvProxyInfo> grvProxyInfo;
 	Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes;
 	HealthMetrics healthMetrics;
 	DatabaseConfiguration configuration;
@@ -548,7 +548,7 @@ struct RatekeeperData {
 
 	bool autoThrottlingEnabled;
 
-	RatekeeperData(UID id, Database db);
+	Ratekeeper(UID id, Database db);
 
 	Future<Void> configurationMonitor();
 	void updateCommitCostEstimation(UIDTransactionTagMap<TransactionCommitCostEstimation> const& costEstimation);

From 49ba96e0bb2c1a5078ffad5e64650d28a57f70c1 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 13:02:09 -0800
Subject: [PATCH 064/210] Clean up ratekeeper includes

---
 fdbserver/Ratekeeper.actor.cpp |  3 +++
 fdbserver/Ratekeeper.h         | 17 ++++++-----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 669b9d08a0..73c500d17d 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -18,7 +18,10 @@
  * limitations under the License.
  */
 
+#include "fdbserver/DataDistribution.actor.h"
 #include "fdbserver/Ratekeeper.h"
+#include "fdbserver/WaitFailure.h"
+
 #include "flow/actorcompiler.h" // must be last include
 
 const char* limitReasonName[] = { "workload",
diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index d29dffb99b..143bdee509 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -18,22 +18,17 @@
  * limitations under the License.
  */
 
-// TODO: Trim down these includes
-#include "fdbserver/WorkerInterface.actor.h"
-#include "flow/IndexedSet.h"
-#include "fdbrpc/FailureMonitor.h"
-#include "fdbrpc/Smoother.h"
-#include "fdbrpc/simulator.h"
+#pragma once
+
+#include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/TagThrottle.actor.h"
+#include "fdbrpc/Smoother.h"
 #include "fdbserver/Knobs.h"
-#include "fdbserver/DataDistribution.actor.h"
 #include "fdbserver/RatekeeperInterface.h"
 #include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/WaitFailure.h"
-
-#pragma once
+#include "fdbserver/TLogInterface.h"
 
 enum limitReason_t {
 	unlimited, // TODO: rename to workload?

From 00f12687c627bc627e7cf9a2a2ff48d7b2d989d5 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 16:03:37 -0800
Subject: [PATCH 065/210] Add TagThrottler class

---
 fdbserver/CMakeLists.txt         |   2 +
 fdbserver/Ratekeeper.actor.cpp   | 161 +++---------
 fdbserver/Ratekeeper.h           | 364 +--------------------------
 fdbserver/TagThrottler.actor.cpp | 140 +++++++++++
 fdbserver/TagThrottler.h         | 412 +++++++++++++++++++++++++++++++
 5 files changed, 583 insertions(+), 496 deletions(-)
 create mode 100644 fdbserver/TagThrottler.actor.cpp
 create mode 100644 fdbserver/TagThrottler.h

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 01324cae05..6b1fbd31dd 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -131,6 +131,8 @@ set(FDBSERVER_SRCS
   storageserver.actor.cpp
   TagPartitionedLogSystem.actor.cpp
   TagPartitionedLogSystem.actor.h
+  TagThrottler.actor.cpp
+  TagThrottler.h
   template_fdb.h
   TCInfo.actor.cpp
   TCInfo.h
diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 73c500d17d..ed8b3f8d56 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbserver/DataDistribution.actor.h"
 #include "fdbserver/Ratekeeper.h"
+#include "fdbserver/TagThrottler.h"
 #include "fdbserver/WaitFailure.h"
 
 #include "flow/actorcompiler.h" // must be last include
@@ -265,114 +266,8 @@ public:
 	}
 
 	ACTOR static Future<Void> monitorThrottlingChanges(Ratekeeper* self) {
-		state bool committed = false;
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
-
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-
-					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
-					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
-
-					if (!committed) {
-						BinaryWriter limitWriter(Unversioned());
-						limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
-						tr.set(tagThrottleLimitKey, limitWriter.toValue());
-					}
-
-					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
-
-					if (autoThrottlingEnabled.get().present() &&
-					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
-						TEST(true); // Auto-throttling disabled
-						if (self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
-						}
-						self->autoThrottlingEnabled = false;
-					} else if (autoThrottlingEnabled.get().present() &&
-					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
-						TEST(true); // Auto-throttling enabled
-						if (!self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
-						}
-						self->autoThrottlingEnabled = true;
-					} else {
-						TEST(true); // Auto-throttling unspecified
-						if (autoThrottlingEnabled.get().present()) {
-							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
-							    .detail("Value", autoThrottlingEnabled.get().get());
-						}
-						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
-						if (!committed)
-							tr.set(tagThrottleAutoEnabledKey,
-							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
-					}
-
-					RkTagThrottleCollection updatedTagThrottles;
-
-					TraceEvent("RatekeeperReadThrottledTags", self->id)
-					    .detail("NumThrottledTags", throttledTagKeys.get().size());
-					for (auto entry : throttledTagKeys.get()) {
-						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
-						TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
-
-						ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
-
-						if (tagValue.expirationTime == 0 ||
-						    tagValue.expirationTime > now() + tagValue.initialDuration) {
-							TEST(true); // Converting tag throttle duration to absolute time
-							tagValue.expirationTime = now() + tagValue.initialDuration;
-							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
-							wr << tagValue;
-							state Value value = wr.toValue();
-
-							tr.set(entry.key, value);
-						}
-
-						if (tagValue.expirationTime > now()) {
-							TransactionTag tag = *tagKey.tags.begin();
-							Optional<ClientTagThrottleLimits> oldLimits =
-							    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
-
-							if (tagKey.throttleType == TagThrottleType::AUTO) {
-								updatedTagThrottles.autoThrottleTag(
-								    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
-								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
-									updatedTagThrottles.busyReadTagCount++;
-								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
-									updatedTagThrottles.busyWriteTagCount++;
-								}
-							} else {
-								updatedTagThrottles.manualThrottleTag(self->id,
-								                                      tag,
-								                                      tagKey.priority,
-								                                      tagValue.tpsRate,
-								                                      tagValue.expirationTime,
-								                                      oldLimits);
-							}
-						}
-					}
-
-					self->throttledTags = std::move(updatedTagThrottles);
-					++self->throttledTagChangeId;
-
-					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
-					wait(tr.commit());
-					committed = true;
-
-					wait(watchFuture);
-					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
-					TEST(true); // Tag throttle changes detected
-					break;
-				} catch (Error& e) {
-					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
-					wait(tr.onError(e));
-				}
-			}
-		}
+		wait(self->tagThrottler->monitorThrottlingChanges());
+		return Void();
 	}
 
 	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
@@ -450,8 +345,8 @@ public:
 					if (p.totalTransactions > 0) {
 						self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
 
-						for (auto tag : req.throttledTagCounts) {
-							self.throttledTags.addRequests(tag.first, tag.second);
+						for (auto const& [tag, count] : req.throttledTagCounts) {
+							self.tagThrottler->addRequests(tag, count);
 						}
 					}
 					if (p.batchTransactions > 0) {
@@ -467,12 +362,12 @@ public:
 					reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
 					reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
 
-					if (p.lastThrottledTagChangeId != self.throttledTagChangeId ||
+					if (p.lastThrottledTagChangeId != self.tagThrottler->getThrottledTagChangeId() ||
 					    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
-						p.lastThrottledTagChangeId = self.throttledTagChangeId;
+						p.lastThrottledTagChangeId = self.tagThrottler->getThrottledTagChangeId();
 						p.lastTagPushTime = now();
 
-						reply.throttledTags = self.throttledTags.getClientRates(self.autoThrottlingEnabled);
+						reply.throttledTags = self.tagThrottler->getClientRates();
 						bool returningTagsToProxy =
 						    reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
 						TEST(returningTagsToProxy); // Returning tag throttles to a proxy
@@ -552,14 +447,14 @@ Ratekeeper::Ratekeeper(UID id, Database db)
     smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
     actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
-    lastBusiestCommitTagPick(0), throttledTagChangeId(0), normalLimits(TransactionPriority::DEFAULT,
-                                                                       "",
-                                                                       SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
-                                                                       SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
-                                                                       SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
-                                                                       SERVER_KNOBS->SPRING_BYTES_TLOG,
-                                                                       SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
-                                                                       SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
+    normalLimits(TransactionPriority::DEFAULT,
+                 "",
+                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
+                 SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
+                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
+                 SERVER_KNOBS->SPRING_BYTES_TLOG,
+                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
+                 SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
     batchLimits(TransactionPriority::BATCH,
                 "Batch",
                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
@@ -567,8 +462,8 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
                 SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
-    autoThrottlingEnabled(false) {
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH) {
+	tagThrottler = std::make_unique<TagThrottler>(this);
 	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
 	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
 }
@@ -1059,21 +954,21 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 		    .detail("LimitingStorageServerVersionLag", limitingVersionLag)
 		    .detail("WorstStorageServerDurabilityLag", worstDurabilityLag)
 		    .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag)
-		    .detail("TagsAutoThrottled", throttledTags.autoThrottleCount())
-		    .detail("TagsAutoThrottledBusyRead", throttledTags.busyReadTagCount)
-		    .detail("TagsAutoThrottledBusyWrite", throttledTags.busyWriteTagCount)
-		    .detail("TagsManuallyThrottled", throttledTags.manualThrottleCount())
-		    .detail("AutoThrottlingEnabled", autoThrottlingEnabled)
+		    .detail("TagsAutoThrottled", tagThrottler->autoThrottleCount())
+		    .detail("TagsAutoThrottledBusyRead", tagThrottler->busyReadTagCount())
+		    .detail("TagsAutoThrottledBusyWrite", tagThrottler->busyWriteTagCount())
+		    .detail("TagsManuallyThrottled", tagThrottler->manualThrottleCount())
+		    .detail("AutoThrottlingEnabled", tagThrottler->isAutoThrottlingEnabled())
 		    .trackLatest(name);
 	}
 }
 
 Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
-	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
-		lastBusiestCommitTagPick = now();
+	if (tagThrottler->getLastBusiestCommitTagPick() == 0) { // the first call should be skipped
+		tagThrottler->updateLastBusiestCommitTagPick();
 		return Void();
 	}
-	double elapsed = now() - lastBusiestCommitTagPick;
+	double elapsed = now() - tagThrottler->getLastBusiestCommitTagPick();
 	// for each SS, select the busiest commit tag from ssTrTagCommitCost
 	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
 		it->value.busiestWriteTag.reset();
@@ -1111,7 +1006,7 @@ Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
 		it->value.totalWriteOps = 0;
 		it->value.totalWriteCosts = 0;
 	}
-	lastBusiestCommitTagPick = now();
+	tagThrottler->updateLastBusiestCommitTagPick();
 	return Void();
 }
 
@@ -1120,7 +1015,7 @@ void Ratekeeper::tryAutoThrottleTag(TransactionTag tag, double rate, double busy
 	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
 	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
 		TEST(true); // Transaction tag auto-throttled
-		Optional<double> clientRate = throttledTags.autoThrottleTag(id, tag, busyness);
+		Optional<double> clientRate = tagThrottler->autoThrottleTag(id, tag, busyness);
 		if (clientRate.present()) {
 			TagSet tags;
 			tags.addTag(tag);
diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index 143bdee509..ce3f170173 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -99,364 +99,6 @@ struct TLogQueueInfo {
 	}
 };
 
-class RkTagThrottleCollection : NonCopyable {
-private:
-	struct RkTagData {
-		Smoother requestRate;
-		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-	};
-
-	struct RkTagThrottleData {
-		ClientTagThrottleLimits limits;
-		Smoother clientRate;
-
-		// Only used by auto-throttles
-		double created = now();
-		double lastUpdated = 0;
-		double lastReduced = now();
-		bool rateSet = false;
-
-		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-
-		double getTargetRate(Optional<double> requestRate) {
-			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
-				return limits.tpsRate;
-			} else {
-				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
-			}
-		}
-
-		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
-			if (limits.expiration > now()) {
-				double targetRate = getTargetRate(requestRate);
-				if (targetRate == std::numeric_limits<double>::max()) {
-					rateSet = false;
-					return targetRate;
-				}
-				if (!rateSet) {
-					rateSet = true;
-					clientRate.reset(targetRate);
-				} else {
-					clientRate.setTotal(targetRate);
-				}
-
-				double rate = clientRate.smoothTotal();
-				ASSERT(rate >= 0);
-				return rate;
-			} else {
-				TEST(true); // Get throttle rate for expired throttle
-				rateSet = false;
-				return Optional<double>();
-			}
-		}
-	};
-
-	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
-
-public:
-	RkTagThrottleCollection() {}
-
-	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	void operator=(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
-		ASSERT(currentBusyness > 0);
-
-		if (targetBusyness < 1) {
-			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
-			return requestRate * targetFraction;
-		} else {
-			return std::numeric_limits<double>::max();
-		}
-	}
-
-	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
-	Optional<double> autoThrottleTag(UID id,
-	                                 TransactionTag const& tag,
-	                                 double fractionalBusyness,
-	                                 Optional<double> tpsRate = Optional<double>(),
-	                                 Optional<double> expiration = Optional<double>()) {
-		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
-		ASSERT(!expiration.present() || expiration.get() > now());
-
-		auto itr = autoThrottledTags.find(tag);
-		bool present = (itr != autoThrottledTags.end());
-		if (!present) {
-			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
-				TEST(true); // Reached auto-throttle limit
-				return Optional<double>();
-			}
-
-			itr = autoThrottledTags.try_emplace(tag).first;
-			initializeTag(tag);
-		} else if (itr->second.limits.expiration <= now()) {
-			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
-			present = false;
-			itr->second = RkTagThrottleData();
-		}
-
-		auto& throttle = itr->second;
-
-		if (!tpsRate.present()) {
-			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
-				tpsRate = std::numeric_limits<double>::max();
-				if (present) {
-					return Optional<double>();
-				}
-			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
-				TEST(true); // Tag auto-throttled too quickly
-				return Optional<double>();
-			} else {
-				tpsRate = computeTargetTpsRate(fractionalBusyness,
-				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
-				                               tagData[tag].requestRate.smoothRate());
-
-				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
-					TEST(true); // Tag auto-throttle rate increase attempt while active
-					return Optional<double>();
-				}
-
-				throttle.lastUpdated = now();
-				if (tpsRate.get() < throttle.limits.tpsRate) {
-					throttle.lastReduced = now();
-				}
-			}
-		}
-		if (!expiration.present()) {
-			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
-		}
-
-		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
-
-		throttle.limits.tpsRate = tpsRate.get();
-		throttle.limits.expiration = expiration.get();
-
-		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
-
-		TraceEvent("RkSetAutoThrottle", id)
-		    .detail("Tag", tag)
-		    .detail("TargetRate", tpsRate.get())
-		    .detail("Expiration", expiration.get() - now())
-		    .detail("ClientRate", clientRate)
-		    .detail("Created", now() - throttle.created)
-		    .detail("LastUpdate", now() - throttle.lastUpdated)
-		    .detail("LastReduced", now() - throttle.lastReduced);
-
-		if (tpsRate.get() != std::numeric_limits<double>::max()) {
-			return tpsRate.get();
-		} else {
-			return Optional<double>();
-		}
-	}
-
-	void manualThrottleTag(UID id,
-	                       TransactionTag const& tag,
-	                       TransactionPriority priority,
-	                       double tpsRate,
-	                       double expiration,
-	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
-		ASSERT(tpsRate >= 0);
-		ASSERT(expiration > now());
-
-		auto& priorityThrottleMap = manualThrottledTags[tag];
-		auto result = priorityThrottleMap.try_emplace(priority);
-		initializeTag(tag);
-		ASSERT(result.second); // Updating to the map is done by copying the whole map
-
-		result.first->second.limits.tpsRate = tpsRate;
-		result.first->second.limits.expiration = expiration;
-
-		if (!oldLimits.present()) {
-			TEST(true); // Transaction tag manually throttled
-			TraceEvent("RatekeeperAddingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
-			TEST(true); // Manual transaction tag throttle updated
-			TraceEvent("RatekeeperUpdatingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		}
-
-		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
-		ASSERT(clientRate.present());
-	}
-
-	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
-	                                                             TransactionPriority priority) {
-		auto itr = manualThrottledTags.find(tag);
-		if (itr != manualThrottledTags.end()) {
-			auto priorityItr = itr->second.find(priority);
-			if (priorityItr != itr->second.end()) {
-				return priorityItr->second.limits;
-			}
-		}
-
-		return Optional<ClientTagThrottleLimits>();
-	}
-
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
-		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
-
-		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
-			bool tagPresent = false;
-
-			double requestRate = tagItr->second.requestRate.smoothRate();
-			auto manualItr = manualThrottledTags.find(tagItr->first);
-			if (manualItr != manualThrottledTags.end()) {
-				Optional<ClientTagThrottleLimits> manualClientRate;
-				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
-				     ++priority) {
-					auto priorityItr = manualItr->second.find(*priority);
-					if (priorityItr != manualItr->second.end()) {
-						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
-						if (!priorityClientRate.present()) {
-							TEST(true); // Manual priority throttle expired
-							priorityItr = manualItr->second.erase(priorityItr);
-						} else {
-							if (!manualClientRate.present() ||
-							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
-								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
-								                                           priorityItr->second.limits.expiration);
-							} else {
-								TEST(true); // Manual throttle overriden by higher priority
-							}
-
-							++priorityItr;
-						}
-					}
-
-					if (manualClientRate.present()) {
-						tagPresent = true;
-						TEST(true); // Using manual throttle
-						clientRates[*priority][tagItr->first] = manualClientRate.get();
-					}
-				}
-
-				if (manualItr->second.empty()) {
-					TEST(true); // All manual throttles expired
-					manualThrottledTags.erase(manualItr);
-					break;
-				}
-			}
-
-			auto autoItr = autoThrottledTags.find(tagItr->first);
-			if (autoItr != autoThrottledTags.end()) {
-				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
-				if (autoClientRate.present()) {
-					double adjustedRate = autoClientRate.get();
-					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
-					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
-						TEST(true); // Tag auto-throttle ramping up
-
-						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
-						if (targetBusyness == 0) {
-							targetBusyness = 0.01;
-						}
-
-						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-						adjustedRate =
-						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
-					}
-
-					tagPresent = true;
-					if (autoThrottlingEnabled) {
-						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
-						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
-						if (!result.second && result.first->second.tpsRate > adjustedRate) {
-							result.first->second =
-							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
-						} else {
-							TEST(true); // Auto throttle overriden by manual throttle
-						}
-						clientRates[TransactionPriority::BATCH][tagItr->first] =
-						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
-					}
-				} else {
-					ASSERT(autoItr->second.limits.expiration <= now());
-					TEST(true); // Auto throttle expired
-					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
-						tagPresent = true;
-					} else {
-						autoThrottledTags.erase(autoItr);
-					}
-				}
-			}
-
-			if (!tagPresent) {
-				TEST(true); // All tag throttles expired
-				tagItr = tagData.erase(tagItr);
-			} else {
-				++tagItr;
-			}
-		}
-
-		return clientRates;
-	}
-
-	void addRequests(TransactionTag const& tag, int requests) {
-		if (requests > 0) {
-			TEST(true); // Requests reported for throttled tag
-
-			auto tagItr = tagData.try_emplace(tag);
-			tagItr.first->second.requestRate.addDelta(requests);
-
-			double requestRate = tagItr.first->second.requestRate.smoothRate();
-
-			auto autoItr = autoThrottledTags.find(tag);
-			if (autoItr != autoThrottledTags.end()) {
-				autoItr->second.updateAndGetClientRate(requestRate);
-			}
-
-			auto manualItr = manualThrottledTags.find(tag);
-			if (manualItr != manualThrottledTags.end()) {
-				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
-				     ++priorityItr) {
-					priorityItr->second.updateAndGetClientRate(requestRate);
-				}
-			}
-		}
-	}
-
-	Optional<double> getRequestRate(TransactionTag const& tag) {
-		auto itr = tagData.find(tag);
-		if (itr != tagData.end()) {
-			return itr->second.requestRate.smoothRate();
-		}
-		return Optional<double>();
-	}
-
-	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
-
-	int64_t manualThrottleCount() const {
-		int64_t count = 0;
-		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
-			count += itr->second.size();
-		}
-
-		return count;
-	}
-
-	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
-	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
-	TransactionTagMap<RkTagData> tagData;
-	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
-};
-
 struct RatekeeperLimits {
 	double tpsLimit;
 	Int64MetricHandle tpsLimitMetric;
@@ -528,10 +170,8 @@ struct Ratekeeper {
 
 	double lastWarning;
 	double lastSSListFetchedTimestamp;
-	double lastBusiestCommitTagPick;
 
-	RkTagThrottleCollection throttledTags;
-	uint64_t throttledTagChangeId;
+	std::unique_ptr<class TagThrottler> tagThrottler;
 
 	RatekeeperLimits normalLimits;
 	RatekeeperLimits batchLimits;
@@ -541,8 +181,6 @@ struct Ratekeeper {
 
 	Future<Void> expiredTagThrottleCleanup;
 
-	bool autoThrottlingEnabled;
-
 	Ratekeeper(UID id, Database db);
 
 	Future<Void> configurationMonitor();
diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
new file mode 100644
index 0000000000..d6b8f668a1
--- /dev/null
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -0,0 +1,140 @@
+/*
+ * TagThrottler.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/TagThrottler.h"
+
+class TagThrottlerImpl {
+public:
+	ACTOR static Future<Void> monitorThrottlingChanges(TagThrottler* self) {
+		state bool committed = false;
+		loop {
+			state ReadYourWritesTransaction tr(self->ratekeeper->db);
+
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
+					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
+
+					if (!committed) {
+						BinaryWriter limitWriter(Unversioned());
+						limitWriter << SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
+						tr.set(tagThrottleLimitKey, limitWriter.toValue());
+					}
+
+					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
+
+					if (autoThrottlingEnabled.get().present() &&
+					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
+						TEST(true); // Auto-throttling disabled
+						if (self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingDisabled", self->ratekeeper->id).log();
+						}
+						self->autoThrottlingEnabled = false;
+					} else if (autoThrottlingEnabled.get().present() &&
+					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
+						TEST(true); // Auto-throttling enabled
+						if (!self->autoThrottlingEnabled) {
+							TraceEvent("AutoTagThrottlingEnabled", self->ratekeeper->id).log();
+						}
+						self->autoThrottlingEnabled = true;
+					} else {
+						TEST(true); // Auto-throttling unspecified
+						if (autoThrottlingEnabled.get().present()) {
+							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->ratekeeper->id)
+							    .detail("Value", autoThrottlingEnabled.get().get());
+						}
+						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
+						if (!committed)
+							tr.set(tagThrottleAutoEnabledKey,
+							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
+					}
+
+					RkTagThrottleCollection updatedTagThrottles;
+
+					TraceEvent("RatekeeperReadThrottledTags", self->ratekeeper->id)
+					    .detail("NumThrottledTags", throttledTagKeys.get().size());
+					for (auto entry : throttledTagKeys.get()) {
+						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
+						TagThrottleValue tagValue = TagThrottleValue::fromValue(entry.value);
+
+						ASSERT(tagKey.tags.size() == 1); // Currently, only 1 tag per throttle is supported
+
+						if (tagValue.expirationTime == 0 ||
+						    tagValue.expirationTime > now() + tagValue.initialDuration) {
+							TEST(true); // Converting tag throttle duration to absolute time
+							tagValue.expirationTime = now() + tagValue.initialDuration;
+							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
+							wr << tagValue;
+							state Value value = wr.toValue();
+
+							tr.set(entry.key, value);
+						}
+
+						if (tagValue.expirationTime > now()) {
+							TransactionTag tag = *tagKey.tags.begin();
+							Optional<ClientTagThrottleLimits> oldLimits =
+							    self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority);
+
+							if (tagKey.throttleType == TagThrottleType::AUTO) {
+								updatedTagThrottles.autoThrottleTag(
+								    self->ratekeeper->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
+								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
+									updatedTagThrottles.busyReadTagCount++;
+								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
+									updatedTagThrottles.busyWriteTagCount++;
+								}
+							} else {
+								updatedTagThrottles.manualThrottleTag(self->ratekeeper->id,
+								                                      tag,
+								                                      tagKey.priority,
+								                                      tagValue.tpsRate,
+								                                      tagValue.expirationTime,
+								                                      oldLimits);
+							}
+						}
+					}
+
+					self->throttledTags = std::move(updatedTagThrottles);
+					++self->throttledTagChangeId;
+
+					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
+					wait(tr.commit());
+					committed = true;
+
+					wait(watchFuture);
+					TraceEvent("RatekeeperThrottleSignaled", self->ratekeeper->id).log();
+					TEST(true); // Tag throttle changes detected
+					break;
+				} catch (Error& e) {
+					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->ratekeeper->id).error(e);
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+}; // class TagThrottlerImpl
+
+Future<Void> TagThrottler::monitorThrottlingChanges() {
+	return TagThrottlerImpl::monitorThrottlingChanges(this);
+}
diff --git a/fdbserver/TagThrottler.h b/fdbserver/TagThrottler.h
new file mode 100644
index 0000000000..f555f2200f
--- /dev/null
+++ b/fdbserver/TagThrottler.h
@@ -0,0 +1,412 @@
+/*
+ * TagThrottler.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/Ratekeeper.h"
+
+class RkTagThrottleCollection : NonCopyable {
+private:
+	struct RkTagData {
+		Smoother requestRate;
+		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+	};
+
+	struct RkTagThrottleData {
+		ClientTagThrottleLimits limits;
+		Smoother clientRate;
+
+		// Only used by auto-throttles
+		double created = now();
+		double lastUpdated = 0;
+		double lastReduced = now();
+		bool rateSet = false;
+
+		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+
+		double getTargetRate(Optional<double> requestRate) {
+			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
+				return limits.tpsRate;
+			} else {
+				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
+			}
+		}
+
+		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
+			if (limits.expiration > now()) {
+				double targetRate = getTargetRate(requestRate);
+				if (targetRate == std::numeric_limits<double>::max()) {
+					rateSet = false;
+					return targetRate;
+				}
+				if (!rateSet) {
+					rateSet = true;
+					clientRate.reset(targetRate);
+				} else {
+					clientRate.setTotal(targetRate);
+				}
+
+				double rate = clientRate.smoothTotal();
+				ASSERT(rate >= 0);
+				return rate;
+			} else {
+				TEST(true); // Get throttle rate for expired throttle
+				rateSet = false;
+				return Optional<double>();
+			}
+		}
+	};
+
+	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
+
+public:
+	RkTagThrottleCollection() {}
+
+	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	void operator=(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
+		ASSERT(currentBusyness > 0);
+
+		if (targetBusyness < 1) {
+			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
+			return requestRate * targetFraction;
+		} else {
+			return std::numeric_limits<double>::max();
+		}
+	}
+
+	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
+	Optional<double> autoThrottleTag(UID id,
+	                                 TransactionTag const& tag,
+	                                 double fractionalBusyness,
+	                                 Optional<double> tpsRate = Optional<double>(),
+	                                 Optional<double> expiration = Optional<double>()) {
+		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
+		ASSERT(!expiration.present() || expiration.get() > now());
+
+		auto itr = autoThrottledTags.find(tag);
+		bool present = (itr != autoThrottledTags.end());
+		if (!present) {
+			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
+				TEST(true); // Reached auto-throttle limit
+				return Optional<double>();
+			}
+
+			itr = autoThrottledTags.try_emplace(tag).first;
+			initializeTag(tag);
+		} else if (itr->second.limits.expiration <= now()) {
+			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
+			present = false;
+			itr->second = RkTagThrottleData();
+		}
+
+		auto& throttle = itr->second;
+
+		if (!tpsRate.present()) {
+			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
+				tpsRate = std::numeric_limits<double>::max();
+				if (present) {
+					return Optional<double>();
+				}
+			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
+				TEST(true); // Tag auto-throttled too quickly
+				return Optional<double>();
+			} else {
+				tpsRate = computeTargetTpsRate(fractionalBusyness,
+				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
+				                               tagData[tag].requestRate.smoothRate());
+
+				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
+					TEST(true); // Tag auto-throttle rate increase attempt while active
+					return Optional<double>();
+				}
+
+				throttle.lastUpdated = now();
+				if (tpsRate.get() < throttle.limits.tpsRate) {
+					throttle.lastReduced = now();
+				}
+			}
+		}
+		if (!expiration.present()) {
+			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
+		}
+
+		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
+
+		throttle.limits.tpsRate = tpsRate.get();
+		throttle.limits.expiration = expiration.get();
+
+		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
+
+		TraceEvent("RkSetAutoThrottle", id)
+		    .detail("Tag", tag)
+		    .detail("TargetRate", tpsRate.get())
+		    .detail("Expiration", expiration.get() - now())
+		    .detail("ClientRate", clientRate)
+		    .detail("Created", now() - throttle.created)
+		    .detail("LastUpdate", now() - throttle.lastUpdated)
+		    .detail("LastReduced", now() - throttle.lastReduced);
+
+		if (tpsRate.get() != std::numeric_limits<double>::max()) {
+			return tpsRate.get();
+		} else {
+			return Optional<double>();
+		}
+	}
+
+	void manualThrottleTag(UID id,
+	                       TransactionTag const& tag,
+	                       TransactionPriority priority,
+	                       double tpsRate,
+	                       double expiration,
+	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
+		ASSERT(tpsRate >= 0);
+		ASSERT(expiration > now());
+
+		auto& priorityThrottleMap = manualThrottledTags[tag];
+		auto result = priorityThrottleMap.try_emplace(priority);
+		initializeTag(tag);
+		ASSERT(result.second); // Updating to the map is done by copying the whole map
+
+		result.first->second.limits.tpsRate = tpsRate;
+		result.first->second.limits.expiration = expiration;
+
+		if (!oldLimits.present()) {
+			TEST(true); // Transaction tag manually throttled
+			TraceEvent("RatekeeperAddingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
+			TEST(true); // Manual transaction tag throttle updated
+			TraceEvent("RatekeeperUpdatingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		}
+
+		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
+		ASSERT(clientRate.present());
+	}
+
+	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
+	                                                             TransactionPriority priority) {
+		auto itr = manualThrottledTags.find(tag);
+		if (itr != manualThrottledTags.end()) {
+			auto priorityItr = itr->second.find(priority);
+			if (priorityItr != itr->second.end()) {
+				return priorityItr->second.limits;
+			}
+		}
+
+		return Optional<ClientTagThrottleLimits>();
+	}
+
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
+		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
+
+		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
+			bool tagPresent = false;
+
+			double requestRate = tagItr->second.requestRate.smoothRate();
+			auto manualItr = manualThrottledTags.find(tagItr->first);
+			if (manualItr != manualThrottledTags.end()) {
+				Optional<ClientTagThrottleLimits> manualClientRate;
+				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
+				     ++priority) {
+					auto priorityItr = manualItr->second.find(*priority);
+					if (priorityItr != manualItr->second.end()) {
+						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
+						if (!priorityClientRate.present()) {
+							TEST(true); // Manual priority throttle expired
+							priorityItr = manualItr->second.erase(priorityItr);
+						} else {
+							if (!manualClientRate.present() ||
+							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
+								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
+								                                           priorityItr->second.limits.expiration);
+							} else {
+								TEST(true); // Manual throttle overriden by higher priority
+							}
+
+							++priorityItr;
+						}
+					}
+
+					if (manualClientRate.present()) {
+						tagPresent = true;
+						TEST(true); // Using manual throttle
+						clientRates[*priority][tagItr->first] = manualClientRate.get();
+					}
+				}
+
+				if (manualItr->second.empty()) {
+					TEST(true); // All manual throttles expired
+					manualThrottledTags.erase(manualItr);
+					break;
+				}
+			}
+
+			auto autoItr = autoThrottledTags.find(tagItr->first);
+			if (autoItr != autoThrottledTags.end()) {
+				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
+				if (autoClientRate.present()) {
+					double adjustedRate = autoClientRate.get();
+					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
+					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
+						TEST(true); // Tag auto-throttle ramping up
+
+						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
+						if (targetBusyness == 0) {
+							targetBusyness = 0.01;
+						}
+
+						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+						adjustedRate =
+						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
+					}
+
+					tagPresent = true;
+					if (autoThrottlingEnabled) {
+						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
+						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
+						if (!result.second && result.first->second.tpsRate > adjustedRate) {
+							result.first->second =
+							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
+						} else {
+							TEST(true); // Auto throttle overriden by manual throttle
+						}
+						clientRates[TransactionPriority::BATCH][tagItr->first] =
+						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
+					}
+				} else {
+					ASSERT(autoItr->second.limits.expiration <= now());
+					TEST(true); // Auto throttle expired
+					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
+						tagPresent = true;
+					} else {
+						autoThrottledTags.erase(autoItr);
+					}
+				}
+			}
+
+			if (!tagPresent) {
+				TEST(true); // All tag throttles expired
+				tagItr = tagData.erase(tagItr);
+			} else {
+				++tagItr;
+			}
+		}
+
+		return clientRates;
+	}
+
+	void addRequests(TransactionTag const& tag, int requests) {
+		if (requests > 0) {
+			TEST(true); // Requests reported for throttled tag
+
+			auto tagItr = tagData.try_emplace(tag);
+			tagItr.first->second.requestRate.addDelta(requests);
+
+			double requestRate = tagItr.first->second.requestRate.smoothRate();
+
+			auto autoItr = autoThrottledTags.find(tag);
+			if (autoItr != autoThrottledTags.end()) {
+				autoItr->second.updateAndGetClientRate(requestRate);
+			}
+
+			auto manualItr = manualThrottledTags.find(tag);
+			if (manualItr != manualThrottledTags.end()) {
+				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
+				     ++priorityItr) {
+					priorityItr->second.updateAndGetClientRate(requestRate);
+				}
+			}
+		}
+	}
+
+	Optional<double> getRequestRate(TransactionTag const& tag) {
+		auto itr = tagData.find(tag);
+		if (itr != tagData.end()) {
+			return itr->second.requestRate.smoothRate();
+		}
+		return Optional<double>();
+	}
+
+	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
+
+	int64_t manualThrottleCount() const {
+		int64_t count = 0;
+		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
+			count += itr->second.size();
+		}
+
+		return count;
+	}
+
+	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
+	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
+	TransactionTagMap<RkTagData> tagData;
+	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
+};
+
+class TagThrottler {
+	friend class TagThrottlerImpl;
+
+	Ratekeeper* ratekeeper;
+	RkTagThrottleCollection throttledTags;
+	uint64_t throttledTagChangeId{ 0 };
+	double lastBusiestCommitTagPick;
+	bool autoThrottlingEnabled{ false };
+
+public:
+	TagThrottler(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper), lastBusiestCommitTagPick(0.0) {}
+	Future<Void> monitorThrottlingChanges();
+	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }
+	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
+		return throttledTags.getClientRates(autoThrottlingEnabled);
+	}
+	int64_t autoThrottleCount() const { return throttledTags.autoThrottleCount(); }
+	uint32_t busyReadTagCount() const { return throttledTags.busyReadTagCount; }
+	uint32_t busyWriteTagCount() const { return throttledTags.busyWriteTagCount; }
+	int64_t manualThrottleCount() const { return throttledTags.manualThrottleCount(); }
+	bool isAutoThrottlingEnabled() const { return autoThrottlingEnabled; }
+	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
+		return throttledTags.autoThrottleTag(id, tag, busyness);
+	}
+
+	void updateLastBusiestCommitTagPick() { lastBusiestCommitTagPick = now(); }
+
+	double getLastBusiestCommitTagPick() const { return lastBusiestCommitTagPick; }
+};

From 796249e99d6b467fe587b714f33b89ef5328cb86 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 16:15:51 -0800
Subject: [PATCH 066/210] Move lastBusiestCommitTagPick field back to
 Ratekeeper

---
 fdbserver/Ratekeeper.actor.cpp | 11 ++++++-----
 fdbserver/Ratekeeper.h         |  2 ++
 fdbserver/TagThrottler.h       |  7 +------
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index ed8b3f8d56..55388a9ba6 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -462,7 +462,8 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
                 SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH) {
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
+    lastBusiestCommitTagPick(0.0) {
 	tagThrottler = std::make_unique<TagThrottler>(this);
 	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
 	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
@@ -964,11 +965,11 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 }
 
 Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
-	if (tagThrottler->getLastBusiestCommitTagPick() == 0) { // the first call should be skipped
-		tagThrottler->updateLastBusiestCommitTagPick();
+	if (lastBusiestCommitTagPick == 0) { // the first call should be skipped
+		lastBusiestCommitTagPick = now();
 		return Void();
 	}
-	double elapsed = now() - tagThrottler->getLastBusiestCommitTagPick();
+	double elapsed = now() - lastBusiestCommitTagPick;
 	// for each SS, select the busiest commit tag from ssTrTagCommitCost
 	for (auto it = storageQueueInfo.begin(); it != storageQueueInfo.end(); ++it) {
 		it->value.busiestWriteTag.reset();
@@ -1006,7 +1007,7 @@ Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
 		it->value.totalWriteOps = 0;
 		it->value.totalWriteCosts = 0;
 	}
-	tagThrottler->updateLastBusiestCommitTagPick();
+	lastBusiestCommitTagPick = now();
 	return Void();
 }
 
diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index ce3f170173..070b4b2d5b 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -181,6 +181,8 @@ struct Ratekeeper {
 
 	Future<Void> expiredTagThrottleCleanup;
 
+	double lastBusiestCommitTagPick;
+
 	Ratekeeper(UID id, Database db);
 
 	Future<Void> configurationMonitor();
diff --git a/fdbserver/TagThrottler.h b/fdbserver/TagThrottler.h
index f555f2200f..3909097769 100644
--- a/fdbserver/TagThrottler.h
+++ b/fdbserver/TagThrottler.h
@@ -386,11 +386,10 @@ class TagThrottler {
 	Ratekeeper* ratekeeper;
 	RkTagThrottleCollection throttledTags;
 	uint64_t throttledTagChangeId{ 0 };
-	double lastBusiestCommitTagPick;
 	bool autoThrottlingEnabled{ false };
 
 public:
-	TagThrottler(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper), lastBusiestCommitTagPick(0.0) {}
+	TagThrottler(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper) {}
 	Future<Void> monitorThrottlingChanges();
 	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }
 	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
@@ -405,8 +404,4 @@ public:
 	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
 		return throttledTags.autoThrottleTag(id, tag, busyness);
 	}
-
-	void updateLastBusiestCommitTagPick() { lastBusiestCommitTagPick = now(); }
-
-	double getLastBusiestCommitTagPick() const { return lastBusiestCommitTagPick; }
 };

From b80f89204a754c86e18700584d415af98c1e4f9f Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 18:17:55 -0800
Subject: [PATCH 067/210] Use PImpl for TagThrottler class

---
 fdbserver/TagThrottler.actor.cpp | 414 ++++++++++++++++++++++++++++++-
 fdbserver/TagThrottler.h         | 391 +----------------------------
 2 files changed, 424 insertions(+), 381 deletions(-)

diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
index d6b8f668a1..c2b86326fd 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -20,9 +20,371 @@
 
 #include "fdbserver/TagThrottler.h"
 
-class TagThrottlerImpl {
+class RkTagThrottleCollection : NonCopyable {
+private:
+	struct RkTagData {
+		Smoother requestRate;
+		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+	};
+
+	struct RkTagThrottleData {
+		ClientTagThrottleLimits limits;
+		Smoother clientRate;
+
+		// Only used by auto-throttles
+		double created = now();
+		double lastUpdated = 0;
+		double lastReduced = now();
+		bool rateSet = false;
+
+		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
+
+		double getTargetRate(Optional<double> requestRate) {
+			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
+				return limits.tpsRate;
+			} else {
+				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
+			}
+		}
+
+		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
+			if (limits.expiration > now()) {
+				double targetRate = getTargetRate(requestRate);
+				if (targetRate == std::numeric_limits<double>::max()) {
+					rateSet = false;
+					return targetRate;
+				}
+				if (!rateSet) {
+					rateSet = true;
+					clientRate.reset(targetRate);
+				} else {
+					clientRate.setTotal(targetRate);
+				}
+
+				double rate = clientRate.smoothTotal();
+				ASSERT(rate >= 0);
+				return rate;
+			} else {
+				TEST(true); // Get throttle rate for expired throttle
+				rateSet = false;
+				return Optional<double>();
+			}
+		}
+	};
+
+	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
+
 public:
-	ACTOR static Future<Void> monitorThrottlingChanges(TagThrottler* self) {
+	RkTagThrottleCollection() {}
+
+	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	void operator=(RkTagThrottleCollection&& other) {
+		autoThrottledTags = std::move(other.autoThrottledTags);
+		manualThrottledTags = std::move(other.manualThrottledTags);
+		tagData = std::move(other.tagData);
+	}
+
+	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
+		ASSERT(currentBusyness > 0);
+
+		if (targetBusyness < 1) {
+			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
+			return requestRate * targetFraction;
+		} else {
+			return std::numeric_limits<double>::max();
+		}
+	}
+
+	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
+	Optional<double> autoThrottleTag(UID id,
+	                                 TransactionTag const& tag,
+	                                 double fractionalBusyness,
+	                                 Optional<double> tpsRate = Optional<double>(),
+	                                 Optional<double> expiration = Optional<double>()) {
+		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
+		ASSERT(!expiration.present() || expiration.get() > now());
+
+		auto itr = autoThrottledTags.find(tag);
+		bool present = (itr != autoThrottledTags.end());
+		if (!present) {
+			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
+				TEST(true); // Reached auto-throttle limit
+				return Optional<double>();
+			}
+
+			itr = autoThrottledTags.try_emplace(tag).first;
+			initializeTag(tag);
+		} else if (itr->second.limits.expiration <= now()) {
+			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
+			present = false;
+			itr->second = RkTagThrottleData();
+		}
+
+		auto& throttle = itr->second;
+
+		if (!tpsRate.present()) {
+			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
+				tpsRate = std::numeric_limits<double>::max();
+				if (present) {
+					return Optional<double>();
+				}
+			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
+				TEST(true); // Tag auto-throttled too quickly
+				return Optional<double>();
+			} else {
+				tpsRate = computeTargetTpsRate(fractionalBusyness,
+				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
+				                               tagData[tag].requestRate.smoothRate());
+
+				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
+					TEST(true); // Tag auto-throttle rate increase attempt while active
+					return Optional<double>();
+				}
+
+				throttle.lastUpdated = now();
+				if (tpsRate.get() < throttle.limits.tpsRate) {
+					throttle.lastReduced = now();
+				}
+			}
+		}
+		if (!expiration.present()) {
+			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
+		}
+
+		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
+
+		throttle.limits.tpsRate = tpsRate.get();
+		throttle.limits.expiration = expiration.get();
+
+		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
+
+		TraceEvent("RkSetAutoThrottle", id)
+		    .detail("Tag", tag)
+		    .detail("TargetRate", tpsRate.get())
+		    .detail("Expiration", expiration.get() - now())
+		    .detail("ClientRate", clientRate)
+		    .detail("Created", now() - throttle.created)
+		    .detail("LastUpdate", now() - throttle.lastUpdated)
+		    .detail("LastReduced", now() - throttle.lastReduced);
+
+		if (tpsRate.get() != std::numeric_limits<double>::max()) {
+			return tpsRate.get();
+		} else {
+			return Optional<double>();
+		}
+	}
+
+	void manualThrottleTag(UID id,
+	                       TransactionTag const& tag,
+	                       TransactionPriority priority,
+	                       double tpsRate,
+	                       double expiration,
+	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
+		ASSERT(tpsRate >= 0);
+		ASSERT(expiration > now());
+
+		auto& priorityThrottleMap = manualThrottledTags[tag];
+		auto result = priorityThrottleMap.try_emplace(priority);
+		initializeTag(tag);
+		ASSERT(result.second); // Updating to the map is done by copying the whole map
+
+		result.first->second.limits.tpsRate = tpsRate;
+		result.first->second.limits.expiration = expiration;
+
+		if (!oldLimits.present()) {
+			TEST(true); // Transaction tag manually throttled
+			TraceEvent("RatekeeperAddingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
+			TEST(true); // Manual transaction tag throttle updated
+			TraceEvent("RatekeeperUpdatingManualThrottle", id)
+			    .detail("Tag", tag)
+			    .detail("Rate", tpsRate)
+			    .detail("Priority", transactionPriorityToString(priority))
+			    .detail("SecondsToExpiration", expiration - now());
+		}
+
+		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
+		ASSERT(clientRate.present());
+	}
+
+	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
+	                                                             TransactionPriority priority) {
+		auto itr = manualThrottledTags.find(tag);
+		if (itr != manualThrottledTags.end()) {
+			auto priorityItr = itr->second.find(priority);
+			if (priorityItr != itr->second.end()) {
+				return priorityItr->second.limits;
+			}
+		}
+
+		return Optional<ClientTagThrottleLimits>();
+	}
+
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
+		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
+
+		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
+			bool tagPresent = false;
+
+			double requestRate = tagItr->second.requestRate.smoothRate();
+			auto manualItr = manualThrottledTags.find(tagItr->first);
+			if (manualItr != manualThrottledTags.end()) {
+				Optional<ClientTagThrottleLimits> manualClientRate;
+				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
+				     ++priority) {
+					auto priorityItr = manualItr->second.find(*priority);
+					if (priorityItr != manualItr->second.end()) {
+						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
+						if (!priorityClientRate.present()) {
+							TEST(true); // Manual priority throttle expired
+							priorityItr = manualItr->second.erase(priorityItr);
+						} else {
+							if (!manualClientRate.present() ||
+							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
+								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
+								                                           priorityItr->second.limits.expiration);
+							} else {
+								TEST(true); // Manual throttle overriden by higher priority
+							}
+
+							++priorityItr;
+						}
+					}
+
+					if (manualClientRate.present()) {
+						tagPresent = true;
+						TEST(true); // Using manual throttle
+						clientRates[*priority][tagItr->first] = manualClientRate.get();
+					}
+				}
+
+				if (manualItr->second.empty()) {
+					TEST(true); // All manual throttles expired
+					manualThrottledTags.erase(manualItr);
+					break;
+				}
+			}
+
+			auto autoItr = autoThrottledTags.find(tagItr->first);
+			if (autoItr != autoThrottledTags.end()) {
+				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
+				if (autoClientRate.present()) {
+					double adjustedRate = autoClientRate.get();
+					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
+					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
+						TEST(true); // Tag auto-throttle ramping up
+
+						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
+						if (targetBusyness == 0) {
+							targetBusyness = 0.01;
+						}
+
+						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+						adjustedRate =
+						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
+					}
+
+					tagPresent = true;
+					if (autoThrottlingEnabled) {
+						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
+						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
+						if (!result.second && result.first->second.tpsRate > adjustedRate) {
+							result.first->second =
+							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
+						} else {
+							TEST(true); // Auto throttle overriden by manual throttle
+						}
+						clientRates[TransactionPriority::BATCH][tagItr->first] =
+						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
+					}
+				} else {
+					ASSERT(autoItr->second.limits.expiration <= now());
+					TEST(true); // Auto throttle expired
+					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
+						tagPresent = true;
+					} else {
+						autoThrottledTags.erase(autoItr);
+					}
+				}
+			}
+
+			if (!tagPresent) {
+				TEST(true); // All tag throttles expired
+				tagItr = tagData.erase(tagItr);
+			} else {
+				++tagItr;
+			}
+		}
+
+		return clientRates;
+	}
+
+	void addRequests(TransactionTag const& tag, int requests) {
+		if (requests > 0) {
+			TEST(true); // Requests reported for throttled tag
+
+			auto tagItr = tagData.try_emplace(tag);
+			tagItr.first->second.requestRate.addDelta(requests);
+
+			double requestRate = tagItr.first->second.requestRate.smoothRate();
+
+			auto autoItr = autoThrottledTags.find(tag);
+			if (autoItr != autoThrottledTags.end()) {
+				autoItr->second.updateAndGetClientRate(requestRate);
+			}
+
+			auto manualItr = manualThrottledTags.find(tag);
+			if (manualItr != manualThrottledTags.end()) {
+				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
+				     ++priorityItr) {
+					priorityItr->second.updateAndGetClientRate(requestRate);
+				}
+			}
+		}
+	}
+
+	Optional<double> getRequestRate(TransactionTag const& tag) {
+		auto itr = tagData.find(tag);
+		if (itr != tagData.end()) {
+			return itr->second.requestRate.smoothRate();
+		}
+		return Optional<double>();
+	}
+
+	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
+
+	int64_t manualThrottleCount() const {
+		int64_t count = 0;
+		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
+			count += itr->second.size();
+		}
+
+		return count;
+	}
+
+	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
+	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
+	TransactionTagMap<RkTagData> tagData;
+	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
+};
+
+class TagThrottlerImpl {
+	Ratekeeper* ratekeeper;
+	RkTagThrottleCollection throttledTags;
+	uint64_t throttledTagChangeId{ 0 };
+	bool autoThrottlingEnabled{ false };
+
+	ACTOR static Future<Void> monitorThrottlingChanges(TagThrottlerImpl* self) {
 		state bool committed = false;
 		loop {
 			state ReadYourWritesTransaction tr(self->ratekeeper->db);
@@ -133,8 +495,54 @@ public:
 		}
 	}
 
+public:
+	TagThrottlerImpl(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper) {}
+	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
+
+	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }
+	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
+		return throttledTags.getClientRates(autoThrottlingEnabled);
+	}
+	int64_t autoThrottleCount() const { return throttledTags.autoThrottleCount(); }
+	uint32_t busyReadTagCount() const { return throttledTags.busyReadTagCount; }
+	uint32_t busyWriteTagCount() const { return throttledTags.busyWriteTagCount; }
+	int64_t manualThrottleCount() const { return throttledTags.manualThrottleCount(); }
+	bool isAutoThrottlingEnabled() const { return autoThrottlingEnabled; }
+	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
+		return throttledTags.autoThrottleTag(id, tag, busyness);
+	}
 }; // class TagThrottlerImpl
 
+TagThrottler::TagThrottler(Ratekeeper* ratekeeper) : impl(PImpl<TagThrottlerImpl>::create(ratekeeper)) {}
+TagThrottler::~TagThrottler() = default;
 Future<Void> TagThrottler::monitorThrottlingChanges() {
-	return TagThrottlerImpl::monitorThrottlingChanges(this);
+	return impl->monitorThrottlingChanges();
+}
+void TagThrottler::addRequests(TransactionTag tag, int count) {
+	impl->addRequests(tag, count);
+}
+uint64_t TagThrottler::getThrottledTagChangeId() const {
+	return impl->getThrottledTagChangeId();
+}
+PrioritizedTransactionTagMap<ClientTagThrottleLimits> TagThrottler::getClientRates() {
+	return impl->getClientRates();
+}
+int64_t TagThrottler::autoThrottleCount() const {
+	return impl->autoThrottleCount();
+}
+uint32_t TagThrottler::busyReadTagCount() const {
+	return impl->busyReadTagCount();
+}
+uint32_t TagThrottler::busyWriteTagCount() const {
+	return impl->busyWriteTagCount();
+}
+int64_t TagThrottler::manualThrottleCount() const {
+	return impl->manualThrottleCount();
+}
+bool TagThrottler::isAutoThrottlingEnabled() const {
+	return impl->isAutoThrottlingEnabled();
+}
+Optional<double> TagThrottler::autoThrottleTag(UID id, TransactionTag tag, double busyness) {
+	return impl->autoThrottleTag(id, tag, busyness);
 }
diff --git a/fdbserver/TagThrottler.h b/fdbserver/TagThrottler.h
index 3909097769..edb077adfe 100644
--- a/fdbserver/TagThrottler.h
+++ b/fdbserver/TagThrottler.h
@@ -20,388 +20,23 @@
 
 #pragma once
 
+#include "fdbclient/PImpl.h"
 #include "fdbserver/Ratekeeper.h"
 
-class RkTagThrottleCollection : NonCopyable {
-private:
-	struct RkTagData {
-		Smoother requestRate;
-		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-	};
-
-	struct RkTagThrottleData {
-		ClientTagThrottleLimits limits;
-		Smoother clientRate;
-
-		// Only used by auto-throttles
-		double created = now();
-		double lastUpdated = 0;
-		double lastReduced = now();
-		bool rateSet = false;
-
-		RkTagThrottleData() : clientRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
-
-		double getTargetRate(Optional<double> requestRate) {
-			if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
-				return limits.tpsRate;
-			} else {
-				return std::min(limits.tpsRate, (limits.tpsRate / requestRate.get()) * clientRate.smoothTotal());
-			}
-		}
-
-		Optional<double> updateAndGetClientRate(Optional<double> requestRate) {
-			if (limits.expiration > now()) {
-				double targetRate = getTargetRate(requestRate);
-				if (targetRate == std::numeric_limits<double>::max()) {
-					rateSet = false;
-					return targetRate;
-				}
-				if (!rateSet) {
-					rateSet = true;
-					clientRate.reset(targetRate);
-				} else {
-					clientRate.setTotal(targetRate);
-				}
-
-				double rate = clientRate.smoothTotal();
-				ASSERT(rate >= 0);
-				return rate;
-			} else {
-				TEST(true); // Get throttle rate for expired throttle
-				rateSet = false;
-				return Optional<double>();
-			}
-		}
-	};
-
-	void initializeTag(TransactionTag const& tag) { tagData.try_emplace(tag); }
-
-public:
-	RkTagThrottleCollection() {}
-
-	RkTagThrottleCollection(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	void operator=(RkTagThrottleCollection&& other) {
-		autoThrottledTags = std::move(other.autoThrottledTags);
-		manualThrottledTags = std::move(other.manualThrottledTags);
-		tagData = std::move(other.tagData);
-	}
-
-	double computeTargetTpsRate(double currentBusyness, double targetBusyness, double requestRate) {
-		ASSERT(currentBusyness > 0);
-
-		if (targetBusyness < 1) {
-			double targetFraction = targetBusyness * (1 - currentBusyness) / ((1 - targetBusyness) * currentBusyness);
-			return requestRate * targetFraction;
-		} else {
-			return std::numeric_limits<double>::max();
-		}
-	}
-
-	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
-	Optional<double> autoThrottleTag(UID id,
-	                                 TransactionTag const& tag,
-	                                 double fractionalBusyness,
-	                                 Optional<double> tpsRate = Optional<double>(),
-	                                 Optional<double> expiration = Optional<double>()) {
-		ASSERT(!tpsRate.present() || tpsRate.get() >= 0);
-		ASSERT(!expiration.present() || expiration.get() > now());
-
-		auto itr = autoThrottledTags.find(tag);
-		bool present = (itr != autoThrottledTags.end());
-		if (!present) {
-			if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
-				TEST(true); // Reached auto-throttle limit
-				return Optional<double>();
-			}
-
-			itr = autoThrottledTags.try_emplace(tag).first;
-			initializeTag(tag);
-		} else if (itr->second.limits.expiration <= now()) {
-			TEST(true); // Re-throttling expired tag that hasn't been cleaned up
-			present = false;
-			itr->second = RkTagThrottleData();
-		}
-
-		auto& throttle = itr->second;
-
-		if (!tpsRate.present()) {
-			if (now() <= throttle.created + SERVER_KNOBS->AUTO_TAG_THROTTLE_START_AGGREGATION_TIME) {
-				tpsRate = std::numeric_limits<double>::max();
-				if (present) {
-					return Optional<double>();
-				}
-			} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
-				TEST(true); // Tag auto-throttled too quickly
-				return Optional<double>();
-			} else {
-				tpsRate = computeTargetTpsRate(fractionalBusyness,
-				                               SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS,
-				                               tagData[tag].requestRate.smoothRate());
-
-				if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
-					TEST(true); // Tag auto-throttle rate increase attempt while active
-					return Optional<double>();
-				}
-
-				throttle.lastUpdated = now();
-				if (tpsRate.get() < throttle.limits.tpsRate) {
-					throttle.lastReduced = now();
-				}
-			}
-		}
-		if (!expiration.present()) {
-			expiration = now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION;
-		}
-
-		ASSERT(tpsRate.present() && tpsRate.get() >= 0);
-
-		throttle.limits.tpsRate = tpsRate.get();
-		throttle.limits.expiration = expiration.get();
-
-		Optional<double> clientRate = throttle.updateAndGetClientRate(getRequestRate(tag));
-
-		TraceEvent("RkSetAutoThrottle", id)
-		    .detail("Tag", tag)
-		    .detail("TargetRate", tpsRate.get())
-		    .detail("Expiration", expiration.get() - now())
-		    .detail("ClientRate", clientRate)
-		    .detail("Created", now() - throttle.created)
-		    .detail("LastUpdate", now() - throttle.lastUpdated)
-		    .detail("LastReduced", now() - throttle.lastReduced);
-
-		if (tpsRate.get() != std::numeric_limits<double>::max()) {
-			return tpsRate.get();
-		} else {
-			return Optional<double>();
-		}
-	}
-
-	void manualThrottleTag(UID id,
-	                       TransactionTag const& tag,
-	                       TransactionPriority priority,
-	                       double tpsRate,
-	                       double expiration,
-	                       Optional<ClientTagThrottleLimits> const& oldLimits) {
-		ASSERT(tpsRate >= 0);
-		ASSERT(expiration > now());
-
-		auto& priorityThrottleMap = manualThrottledTags[tag];
-		auto result = priorityThrottleMap.try_emplace(priority);
-		initializeTag(tag);
-		ASSERT(result.second); // Updating to the map is done by copying the whole map
-
-		result.first->second.limits.tpsRate = tpsRate;
-		result.first->second.limits.expiration = expiration;
-
-		if (!oldLimits.present()) {
-			TEST(true); // Transaction tag manually throttled
-			TraceEvent("RatekeeperAddingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
-			TEST(true); // Manual transaction tag throttle updated
-			TraceEvent("RatekeeperUpdatingManualThrottle", id)
-			    .detail("Tag", tag)
-			    .detail("Rate", tpsRate)
-			    .detail("Priority", transactionPriorityToString(priority))
-			    .detail("SecondsToExpiration", expiration - now());
-		}
-
-		Optional<double> clientRate = result.first->second.updateAndGetClientRate(getRequestRate(tag));
-		ASSERT(clientRate.present());
-	}
-
-	Optional<ClientTagThrottleLimits> getManualTagThrottleLimits(TransactionTag const& tag,
-	                                                             TransactionPriority priority) {
-		auto itr = manualThrottledTags.find(tag);
-		if (itr != manualThrottledTags.end()) {
-			auto priorityItr = itr->second.find(priority);
-			if (priorityItr != itr->second.end()) {
-				return priorityItr->second.limits;
-			}
-		}
-
-		return Optional<ClientTagThrottleLimits>();
-	}
-
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates(bool autoThrottlingEnabled) {
-		PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientRates;
-
-		for (auto tagItr = tagData.begin(); tagItr != tagData.end();) {
-			bool tagPresent = false;
-
-			double requestRate = tagItr->second.requestRate.smoothRate();
-			auto manualItr = manualThrottledTags.find(tagItr->first);
-			if (manualItr != manualThrottledTags.end()) {
-				Optional<ClientTagThrottleLimits> manualClientRate;
-				for (auto priority = allTransactionPriorities.rbegin(); !(priority == allTransactionPriorities.rend());
-				     ++priority) {
-					auto priorityItr = manualItr->second.find(*priority);
-					if (priorityItr != manualItr->second.end()) {
-						Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
-						if (!priorityClientRate.present()) {
-							TEST(true); // Manual priority throttle expired
-							priorityItr = manualItr->second.erase(priorityItr);
-						} else {
-							if (!manualClientRate.present() ||
-							    manualClientRate.get().tpsRate > priorityClientRate.get()) {
-								manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
-								                                           priorityItr->second.limits.expiration);
-							} else {
-								TEST(true); // Manual throttle overriden by higher priority
-							}
-
-							++priorityItr;
-						}
-					}
-
-					if (manualClientRate.present()) {
-						tagPresent = true;
-						TEST(true); // Using manual throttle
-						clientRates[*priority][tagItr->first] = manualClientRate.get();
-					}
-				}
-
-				if (manualItr->second.empty()) {
-					TEST(true); // All manual throttles expired
-					manualThrottledTags.erase(manualItr);
-					break;
-				}
-			}
-
-			auto autoItr = autoThrottledTags.find(tagItr->first);
-			if (autoItr != autoThrottledTags.end()) {
-				Optional<double> autoClientRate = autoItr->second.updateAndGetClientRate(requestRate);
-				if (autoClientRate.present()) {
-					double adjustedRate = autoClientRate.get();
-					double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
-					                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-					if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
-						TEST(true); // Tag auto-throttle ramping up
-
-						double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
-						if (targetBusyness == 0) {
-							targetBusyness = 0.01;
-						}
-
-						double rampLocation = (now() - rampStartTime) / SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-						adjustedRate =
-						    computeTargetTpsRate(targetBusyness, pow(targetBusyness, 1 - rampLocation), adjustedRate);
-					}
-
-					tagPresent = true;
-					if (autoThrottlingEnabled) {
-						auto result = clientRates[TransactionPriority::DEFAULT].try_emplace(
-						    tagItr->first, adjustedRate, autoItr->second.limits.expiration);
-						if (!result.second && result.first->second.tpsRate > adjustedRate) {
-							result.first->second =
-							    ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
-						} else {
-							TEST(true); // Auto throttle overriden by manual throttle
-						}
-						clientRates[TransactionPriority::BATCH][tagItr->first] =
-						    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
-					}
-				} else {
-					ASSERT(autoItr->second.limits.expiration <= now());
-					TEST(true); // Auto throttle expired
-					if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
-						tagPresent = true;
-					} else {
-						autoThrottledTags.erase(autoItr);
-					}
-				}
-			}
-
-			if (!tagPresent) {
-				TEST(true); // All tag throttles expired
-				tagItr = tagData.erase(tagItr);
-			} else {
-				++tagItr;
-			}
-		}
-
-		return clientRates;
-	}
-
-	void addRequests(TransactionTag const& tag, int requests) {
-		if (requests > 0) {
-			TEST(true); // Requests reported for throttled tag
-
-			auto tagItr = tagData.try_emplace(tag);
-			tagItr.first->second.requestRate.addDelta(requests);
-
-			double requestRate = tagItr.first->second.requestRate.smoothRate();
-
-			auto autoItr = autoThrottledTags.find(tag);
-			if (autoItr != autoThrottledTags.end()) {
-				autoItr->second.updateAndGetClientRate(requestRate);
-			}
-
-			auto manualItr = manualThrottledTags.find(tag);
-			if (manualItr != manualThrottledTags.end()) {
-				for (auto priorityItr = manualItr->second.begin(); priorityItr != manualItr->second.end();
-				     ++priorityItr) {
-					priorityItr->second.updateAndGetClientRate(requestRate);
-				}
-			}
-		}
-	}
-
-	Optional<double> getRequestRate(TransactionTag const& tag) {
-		auto itr = tagData.find(tag);
-		if (itr != tagData.end()) {
-			return itr->second.requestRate.smoothRate();
-		}
-		return Optional<double>();
-	}
-
-	int64_t autoThrottleCount() const { return autoThrottledTags.size(); }
-
-	int64_t manualThrottleCount() const {
-		int64_t count = 0;
-		for (auto itr = manualThrottledTags.begin(); itr != manualThrottledTags.end(); ++itr) {
-			count += itr->second.size();
-		}
-
-		return count;
-	}
-
-	TransactionTagMap<RkTagThrottleData> autoThrottledTags;
-	TransactionTagMap<std::map<TransactionPriority, RkTagThrottleData>> manualThrottledTags;
-	TransactionTagMap<RkTagData> tagData;
-	uint32_t busyReadTagCount = 0, busyWriteTagCount = 0;
-};
-
 class TagThrottler {
-	friend class TagThrottlerImpl;
-
-	Ratekeeper* ratekeeper;
-	RkTagThrottleCollection throttledTags;
-	uint64_t throttledTagChangeId{ 0 };
-	bool autoThrottlingEnabled{ false };
+	PImpl<class TagThrottlerImpl> impl;
 
 public:
-	TagThrottler(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper) {}
+	TagThrottler(Ratekeeper* ratekeeper);
+	~TagThrottler();
 	Future<Void> monitorThrottlingChanges();
-	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }
-	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
-		return throttledTags.getClientRates(autoThrottlingEnabled);
-	}
-	int64_t autoThrottleCount() const { return throttledTags.autoThrottleCount(); }
-	uint32_t busyReadTagCount() const { return throttledTags.busyReadTagCount; }
-	uint32_t busyWriteTagCount() const { return throttledTags.busyWriteTagCount; }
-	int64_t manualThrottleCount() const { return throttledTags.manualThrottleCount(); }
-	bool isAutoThrottlingEnabled() const { return autoThrottlingEnabled; }
-	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
-		return throttledTags.autoThrottleTag(id, tag, busyness);
-	}
+	void addRequests(TransactionTag tag, int count);
+	uint64_t getThrottledTagChangeId() const;
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates();
+	int64_t autoThrottleCount() const;
+	uint32_t busyReadTagCount() const;
+	uint32_t busyWriteTagCount() const;
+	int64_t manualThrottleCount() const;
+	bool isAutoThrottlingEnabled() const;
+	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness);
 };

From 58669717f17cd2490d7469a1a5522d1f389095a0 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 18:54:12 -0800
Subject: [PATCH 068/210] Move tryAutoThrottleTag method to TagThrottler

---
 fdbserver/Ratekeeper.actor.cpp   | 46 +------------------------
 fdbserver/TagThrottler.actor.cpp | 59 +++++++++++++++++++++++++++++---
 fdbserver/TagThrottler.h         |  2 +-
 3 files changed, 56 insertions(+), 51 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 55388a9ba6..1e0ca519b5 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -573,7 +573,7 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 		double targetRateRatio = std::min((storageQueue - targetBytes + springBytes) / (double)springBytes, 2.0);
 
 		if (limits->priority == TransactionPriority::DEFAULT) {
-			tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag);
+			addActor.send(tagThrottler->tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag));
 		}
 
 		double inputRate = ss.smoothInputBytes.smoothRate();
@@ -1011,50 +1011,6 @@ Future<Void> Ratekeeper::refreshStorageServerCommitCost() {
 	return Void();
 }
 
-void Ratekeeper::tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
-	// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
-	// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
-	if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
-		TEST(true); // Transaction tag auto-throttled
-		Optional<double> clientRate = tagThrottler->autoThrottleTag(id, tag, busyness);
-		if (clientRate.present()) {
-			TagSet tags;
-			tags.addTag(tag);
-
-			Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(db.getPtr());
-			addActor.send(ThrottleApi::throttleTags(dbRef,
-			                                        tags,
-			                                        clientRate.get(),
-			                                        SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                        TagThrottleType::AUTO,
-			                                        TransactionPriority::DEFAULT,
-			                                        now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
-			                                        reason));
-		}
-	}
-}
-
-void Ratekeeper::tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
-	// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most
-	// of situation, this works. More indicators besides queue size and durability lag could be investigated in the
-	// future
-	if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
-	    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
-		if (ss.busiestWriteTag.present()) {
-			tryAutoThrottleTag(ss.busiestWriteTag.get(),
-			                   ss.busiestWriteTagRate,
-			                   ss.busiestWriteTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_WRITE);
-		}
-		if (ss.busiestReadTag.present()) {
-			tryAutoThrottleTag(ss.busiestReadTag.get(),
-			                   ss.busiestReadTagRate,
-			                   ss.busiestReadTagFractionalBusyness,
-			                   TagThrottledReason::BUSY_READ);
-		}
-	}
-}
-
 ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	wait(Ratekeeper::run(rkInterf, dbInfo));
 	return Void();
diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
index c2b86326fd..b772809eb6 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -21,7 +21,6 @@
 #include "fdbserver/TagThrottler.h"
 
 class RkTagThrottleCollection : NonCopyable {
-private:
 	struct RkTagData {
 		Smoother requestRate;
 		RkTagData() : requestRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {}
@@ -495,6 +494,34 @@ class TagThrottlerImpl {
 		}
 	}
 
+	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
+		return throttledTags.autoThrottleTag(id, tag, busyness);
+	}
+
+	Future<Void> tryAutoThrottleTag(TransactionTag tag, double rate, double busyness, TagThrottledReason reason) {
+		// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
+		// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
+		if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
+			TEST(true); // Transaction tag auto-throttled
+			Optional<double> clientRate = autoThrottleTag(ratekeeper->id, tag, busyness);
+			if (clientRate.present()) {
+				TagSet tags;
+				tags.addTag(tag);
+
+				Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(ratekeeper->db.getPtr());
+				return ThrottleApi::throttleTags(dbRef,
+				                                 tags,
+				                                 clientRate.get(),
+				                                 SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+				                                 TagThrottleType::AUTO,
+				                                 TransactionPriority::DEFAULT,
+				                                 now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION,
+				                                 reason);
+			}
+		}
+		return Void();
+	}
+
 public:
 	TagThrottlerImpl(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper) {}
 	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
@@ -509,9 +536,29 @@ public:
 	uint32_t busyWriteTagCount() const { return throttledTags.busyWriteTagCount; }
 	int64_t manualThrottleCount() const { return throttledTags.manualThrottleCount(); }
 	bool isAutoThrottlingEnabled() const { return autoThrottlingEnabled; }
-	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness) {
-		return throttledTags.autoThrottleTag(id, tag, busyness);
+
+	Future<Void> tryAutoThrottleTag(StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) {
+		// NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In
+		// most of situation, this works. More indicators besides queue size and durability lag could be investigated in
+		// the future
+		if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES ||
+		    storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) {
+			if (ss.busiestWriteTag.present()) {
+				return tryAutoThrottleTag(ss.busiestWriteTag.get(),
+				                          ss.busiestWriteTagRate,
+				                          ss.busiestWriteTagFractionalBusyness,
+				                          TagThrottledReason::BUSY_WRITE);
+			}
+			if (ss.busiestReadTag.present()) {
+				return tryAutoThrottleTag(ss.busiestReadTag.get(),
+				                          ss.busiestReadTagRate,
+				                          ss.busiestReadTagFractionalBusyness,
+				                          TagThrottledReason::BUSY_READ);
+			}
+		}
+		return Void();
 	}
+
 }; // class TagThrottlerImpl
 
 TagThrottler::TagThrottler(Ratekeeper* ratekeeper) : impl(PImpl<TagThrottlerImpl>::create(ratekeeper)) {}
@@ -543,6 +590,8 @@ int64_t TagThrottler::manualThrottleCount() const {
 bool TagThrottler::isAutoThrottlingEnabled() const {
 	return impl->isAutoThrottlingEnabled();
 }
-Optional<double> TagThrottler::autoThrottleTag(UID id, TransactionTag tag, double busyness) {
-	return impl->autoThrottleTag(id, tag, busyness);
+Future<Void> TagThrottler::tryAutoThrottleTag(StorageQueueInfo& ss,
+                                              int64_t storageQueue,
+                                              int64_t storageDurabilityLag) {
+	return impl->tryAutoThrottleTag(ss, storageQueue, storageDurabilityLag);
 }
diff --git a/fdbserver/TagThrottler.h b/fdbserver/TagThrottler.h
index edb077adfe..8f4e682ae7 100644
--- a/fdbserver/TagThrottler.h
+++ b/fdbserver/TagThrottler.h
@@ -38,5 +38,5 @@ public:
 	uint32_t busyWriteTagCount() const;
 	int64_t manualThrottleCount() const;
 	bool isAutoThrottlingEnabled() const;
-	Optional<double> autoThrottleTag(UID id, TransactionTag tag, double busyness);
+	Future<Void> tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
 };

From 182fb8de3dd8b489efc00a6915568f541435deb8 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 18:57:16 -0800
Subject: [PATCH 069/210] Change type of TagThrottlerImpl::ratekeeper

---
 fdbserver/TagThrottler.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
index b772809eb6..6a0372ce7f 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -378,7 +378,7 @@ public:
 };
 
 class TagThrottlerImpl {
-	Ratekeeper* ratekeeper;
+	Ratekeeper const* ratekeeper;
 	RkTagThrottleCollection throttledTags;
 	uint64_t throttledTagChangeId{ 0 };
 	bool autoThrottlingEnabled{ false };
@@ -523,7 +523,7 @@ class TagThrottlerImpl {
 	}
 
 public:
-	TagThrottlerImpl(Ratekeeper* ratekeeper) : ratekeeper(ratekeeper) {}
+	TagThrottlerImpl(Ratekeeper const* ratekeeper) : ratekeeper(ratekeeper) {}
 	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
 
 	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }

From 0f4c808f37b3dd8b37ec9d10569a300a19354513 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 19:08:23 -0800
Subject: [PATCH 070/210] Remove TagThrottler::ratekeeper field

---
 fdbserver/Ratekeeper.actor.cpp   |  2 +-
 fdbserver/TagThrottler.actor.cpp | 29 +++++++++++++++--------------
 fdbserver/TagThrottler.h         |  2 +-
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 1e0ca519b5..84fdf39877 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -464,7 +464,7 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
                 SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
     lastBusiestCommitTagPick(0.0) {
-	tagThrottler = std::make_unique<TagThrottler>(this);
+	tagThrottler = std::make_unique<TagThrottler>(db, id);
 	expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
 	                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
 }
diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
index 6a0372ce7f..2e215a7c9b 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -378,7 +378,8 @@ public:
 };
 
 class TagThrottlerImpl {
-	Ratekeeper const* ratekeeper;
+	Database db;
+	UID id;
 	RkTagThrottleCollection throttledTags;
 	uint64_t throttledTagChangeId{ 0 };
 	bool autoThrottlingEnabled{ false };
@@ -386,7 +387,7 @@ class TagThrottlerImpl {
 	ACTOR static Future<Void> monitorThrottlingChanges(TagThrottlerImpl* self) {
 		state bool committed = false;
 		loop {
-			state ReadYourWritesTransaction tr(self->ratekeeper->db);
+			state ReadYourWritesTransaction tr(self->db);
 
 			loop {
 				try {
@@ -408,20 +409,20 @@ class TagThrottlerImpl {
 					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
 						TEST(true); // Auto-throttling disabled
 						if (self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingDisabled", self->ratekeeper->id).log();
+							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
 						}
 						self->autoThrottlingEnabled = false;
 					} else if (autoThrottlingEnabled.get().present() &&
 					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
 						TEST(true); // Auto-throttling enabled
 						if (!self->autoThrottlingEnabled) {
-							TraceEvent("AutoTagThrottlingEnabled", self->ratekeeper->id).log();
+							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
 						}
 						self->autoThrottlingEnabled = true;
 					} else {
 						TEST(true); // Auto-throttling unspecified
 						if (autoThrottlingEnabled.get().present()) {
-							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->ratekeeper->id)
+							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
 							    .detail("Value", autoThrottlingEnabled.get().get());
 						}
 						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
@@ -432,7 +433,7 @@ class TagThrottlerImpl {
 
 					RkTagThrottleCollection updatedTagThrottles;
 
-					TraceEvent("RatekeeperReadThrottledTags", self->ratekeeper->id)
+					TraceEvent("RatekeeperReadThrottledTags", self->id)
 					    .detail("NumThrottledTags", throttledTagKeys.get().size());
 					for (auto entry : throttledTagKeys.get()) {
 						TagThrottleKey tagKey = TagThrottleKey::fromKey(entry.key);
@@ -458,14 +459,14 @@ class TagThrottlerImpl {
 
 							if (tagKey.throttleType == TagThrottleType::AUTO) {
 								updatedTagThrottles.autoThrottleTag(
-								    self->ratekeeper->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
+								    self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime);
 								if (tagValue.reason == TagThrottledReason::BUSY_READ) {
 									updatedTagThrottles.busyReadTagCount++;
 								} else if (tagValue.reason == TagThrottledReason::BUSY_WRITE) {
 									updatedTagThrottles.busyWriteTagCount++;
 								}
 							} else {
-								updatedTagThrottles.manualThrottleTag(self->ratekeeper->id,
+								updatedTagThrottles.manualThrottleTag(self->id,
 								                                      tag,
 								                                      tagKey.priority,
 								                                      tagValue.tpsRate,
@@ -483,11 +484,11 @@ class TagThrottlerImpl {
 					committed = true;
 
 					wait(watchFuture);
-					TraceEvent("RatekeeperThrottleSignaled", self->ratekeeper->id).log();
+					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
 					TEST(true); // Tag throttle changes detected
 					break;
 				} catch (Error& e) {
-					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->ratekeeper->id).error(e);
+					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
 					wait(tr.onError(e));
 				}
 			}
@@ -503,12 +504,12 @@ class TagThrottlerImpl {
 		// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
 		if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
 			TEST(true); // Transaction tag auto-throttled
-			Optional<double> clientRate = autoThrottleTag(ratekeeper->id, tag, busyness);
+			Optional<double> clientRate = autoThrottleTag(id, tag, busyness);
 			if (clientRate.present()) {
 				TagSet tags;
 				tags.addTag(tag);
 
-				Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(ratekeeper->db.getPtr());
+				Reference<DatabaseContext> dbRef = Reference<DatabaseContext>::addRef(db.getPtr());
 				return ThrottleApi::throttleTags(dbRef,
 				                                 tags,
 				                                 clientRate.get(),
@@ -523,7 +524,7 @@ class TagThrottlerImpl {
 	}
 
 public:
-	TagThrottlerImpl(Ratekeeper const* ratekeeper) : ratekeeper(ratekeeper) {}
+	TagThrottlerImpl(Database db, UID id) : db(db), id(id) {}
 	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
 
 	void addRequests(TransactionTag tag, int count) { throttledTags.addRequests(tag, count); }
@@ -561,7 +562,7 @@ public:
 
 }; // class TagThrottlerImpl
 
-TagThrottler::TagThrottler(Ratekeeper* ratekeeper) : impl(PImpl<TagThrottlerImpl>::create(ratekeeper)) {}
+TagThrottler::TagThrottler(Database db, UID id) : impl(PImpl<TagThrottlerImpl>::create(db, id)) {}
 TagThrottler::~TagThrottler() = default;
 Future<Void> TagThrottler::monitorThrottlingChanges() {
 	return impl->monitorThrottlingChanges();
diff --git a/fdbserver/TagThrottler.h b/fdbserver/TagThrottler.h
index 8f4e682ae7..4070162545 100644
--- a/fdbserver/TagThrottler.h
+++ b/fdbserver/TagThrottler.h
@@ -27,7 +27,7 @@ class TagThrottler {
 	PImpl<class TagThrottlerImpl> impl;
 
 public:
-	TagThrottler(Ratekeeper* ratekeeper);
+	TagThrottler(Database db, UID id);
 	~TagThrottler();
 	Future<Void> monitorThrottlingChanges();
 	void addRequests(TransactionTag tag, int count);

From 47d549245946d85f727b4f6b359131d90a252cb7 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Mon, 14 Feb 2022 19:28:19 -0800
Subject: [PATCH 071/210] Make many fields and methods of Ratekeeper private

---
 fdbserver/Ratekeeper.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h
index 070b4b2d5b..8552eeb521 100644
--- a/fdbserver/Ratekeeper.h
+++ b/fdbserver/Ratekeeper.h
@@ -139,7 +139,9 @@ struct RatekeeperLimits {
 	    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
 };
 
-struct Ratekeeper {
+class Ratekeeper {
+	friend class RatekeeperImpl;
+
 	// Differentiate from GrvProxyInfo in DatabaseContext.h
 	struct GrvProxyInfo {
 		int64_t totalTransactions;
@@ -200,5 +202,6 @@ struct Ratekeeper {
 	void tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
 	Future<Void> monitorThrottlingChanges();
 
+public:
 	static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 };

From 595f50ce2680299e96b5b50b0475ee23b87e8512 Mon Sep 17 00:00:00 2001
From: vikasgupta8 <vikas.gupta8@ibm.com>
Date: Tue, 15 Feb 2022 13:03:12 +0000
Subject: [PATCH 072/210] indentation corrected

---
 cmake/ConfigureCompiler.cmake | 4 ++--
 flow/Platform.h               | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 586753bdc5..83ed75e873 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -219,7 +219,7 @@ else()
   endif()
   if(STATIC_LINK_LIBCXX)
     if (NOT USE_LIBCXX AND NOT APPLE)
-	    add_link_options(-static-libstdc++ -static-libgcc)
+	add_link_options(-static-libstdc++ -static-libgcc)
     endif()
   endif()
   # # Instruction sets we require to be supported by the CPU
@@ -350,7 +350,7 @@ else()
   endif()
 
   if (CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
-      add_compile_options(-m64 -mcpu=power9 -mtune=power9 -DNO_WARN_X86_INTRINSICS)
+    add_compile_options(-m64 -mcpu=power9 -mtune=power9 -DNO_WARN_X86_INTRINSICS)
   endif()
   # Check whether we can use dtrace probes
   include(CheckSymbolExists)
diff --git a/flow/Platform.h b/flow/Platform.h
index 505e475c58..3a8d75a3f5 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -842,7 +842,7 @@ static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
 #endif
 #endif
 
-#if defined(__aarch64__) || defined(__powerpc64__)
+#if defined(__aarch64__)
 #define _MM_HINT_T0 0 /* dummy -- not used */
 #endif
 

From 6f1c3d50bb2060d821533b1751b57bcd86dcb5c1 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 15 Feb 2022 15:08:53 -0500
Subject: [PATCH 073/210] add debug traces for testing

---
 fdbclient/NativeAPI.actor.cpp      | 2 ++
 fdbserver/GrvProxyServer.actor.cpp | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 8070c23a66..5fe063616b 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5968,9 +5968,11 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	trState->cx->lastProxyRequestTime = trState->startTime;
 	trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
 	if (rep.rkBatchThrottled) {
+		TraceEvent("DebugGrvLastRkBatchThrottled").detail("ReplyTime", replyTime);
 		trState->cx->lastRkBatchThrottleTime = replyTime;
 	}
 	if (rep.rkDefaultThrottled) {
+		TraceEvent("DebugGrvLastRkDefaultThrottled").detail("ReplyTime", replyTime);
 		trState->cx->lastRkDefaultThrottleTime = replyTime;
 	}
 	trState->cx->GRVLatencies.addSample(latency);
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 6fe559d02d..4aae181e57 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -659,6 +659,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		if (stats->lastBatchQueueThrottled) {
 			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 			if (now() - stats->batchThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
+				TraceEvent("DebugGrvProxyBatchThrottled").detail("StartTime", stats->batchThrottleStartTime);
 				reply.rkBatchThrottled = true;
 			}
 		}
@@ -668,6 +669,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 				// Consider the batch queue throttled if the default is throttled
 				// to deal with a potential lull in activity for that priority.
 				// Avoids mistakenly thinking batch is unthrottled while default is still throttled.
+				TraceEvent("DebugGrvProxyDefaultThrottled").detail("StartTime", stats->defaultThrottleStartTime);
 				reply.rkBatchThrottled = true;
 				reply.rkDefaultThrottled = true;
 			}
@@ -869,12 +871,14 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			requestsToStart++;
 		}
 		if (!batchQueue.empty()) {
+			TraceEvent("DebugGrvProxyBatchQueueEmpty").detail("Size", batchQueue.size());
 			grvProxyData->stats.lastBatchQueueThrottled = true;
 			grvProxyData->stats.batchThrottleStartTime = now();
 		} else {
 			grvProxyData->stats.lastBatchQueueThrottled = false;
 		}
 		if (!defaultQueue.empty()) {
+			TraceEvent("DebugGrvProxyDefaultQueueEmpty").detail("Size", defaultQueue.size());
 			grvProxyData->stats.lastDefaultQueueThrottled = true;
 			grvProxyData->stats.defaultThrottleStartTime = now();
 		} else {

From fe03e6f82289b7634a58da6468d5a2ec37504617 Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Tue, 15 Feb 2022 22:43:06 -0800
Subject: [PATCH 074/210] Introduce a new server knob and use it to test if
 storage servers are near the min bar for available space

---
 fdbclient/ServerKnobs.cpp            |  1 +
 fdbclient/ServerKnobs.h              |  1 +
 fdbserver/DDTeamCollection.actor.cpp |  6 +----
 fdbserver/TCInfo.actor.cpp           | 34 +++++++++++++++++++++++++++-
 fdbserver/TCInfo.h                   |  4 ++++
 5 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 752839ccc6..a24100870e 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -579,6 +579,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( MIN_AVAILABLE_SPACE,                                   1e8 );
 	init( MIN_AVAILABLE_SPACE_RATIO,                            0.05 );
+	init( AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER,           0.03 );
 	init( TARGET_AVAILABLE_SPACE_RATIO,                         0.30 );
 	init( AVAILABLE_SPACE_UPDATE_DELAY,                          5.0 );
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index a813bad097..1eac4c27e2 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -525,6 +525,7 @@ public:
 
 	int64_t MIN_AVAILABLE_SPACE;
 	double MIN_AVAILABLE_SPACE_RATIO;
+	double AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
 	double TARGET_AVAILABLE_SPACE_RATIO;
 	double AVAILABLE_SPACE_UPDATE_DELAY;
 
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index c6dc10b0f1..b0effff1d9 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -5671,11 +5671,7 @@ TEST_CASE("/DataDistribution/GetTeam/ServerUtilizationNearCutoff") {
 
 	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
 
-	std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-	ASSERT(expectedServers == selectedServers);
+	ASSERT(!resTeam.first.present());
 
 	return Void();
 }
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index a430b0d5f7..6d4e65afa4 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -138,6 +138,23 @@ TCServerInfo::TCServerInfo(StorageServerInterface ssi,
 	}
 }
 
+bool TCServerInfo::hasHealthyAvailableSpace(double minAvailableSpaceRatio) const {
+	ASSERT(serverMetrics.present());
+
+	auto& metrics = serverMetrics.get();
+	ASSERT(metrics.available.bytes >= 0);
+	ASSERT(metrics.capacity.bytes >= 0);
+
+	double availableSpaceRatio;
+	if (metrics.capacity.bytes == 0) {
+		availableSpaceRatio = 0;
+	} else {
+		availableSpaceRatio = (((double)metrics.available.bytes) / metrics.capacity.bytes);
+	}
+
+	return availableSpaceRatio >= minAvailableSpaceRatio;
+}
+
 Future<Void> TCServerInfo::updateServerMetrics() {
 	return TCServerInfoImpl::updateServerMetrics(this);
 }
@@ -319,8 +336,23 @@ double TCTeamInfo::getMinAvailableSpaceRatio(bool includeInFlight) const {
 	return minRatio;
 }
 
+bool TCTeamInfo::allServersHaveHealthyAvailableSpace() const {
+	bool result = true;
+	double minAvailableSpaceRatio =
+	    SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF + SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
+	for (const auto& server : servers) {
+		if (!server->serverMetrics.present() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) {
+			result = false;
+			break;
+		}
+	}
+
+	return result;
+}
+
 bool TCTeamInfo::hasHealthyAvailableSpace(double minRatio) const {
-	return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE;
+	return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE &&
+	       allServersHaveHealthyAvailableSpace();
 }
 
 bool TCTeamInfo::isOptimal() const {
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index b64e415204..101dd845e9 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -71,6 +71,8 @@ public:
 		return (storeType == configStoreType || storeType == KeyValueStoreType::END);
 	}
 
+	bool hasHealthyAvailableSpace(double minAvailableSpaceRatio) const;
+
 	Future<Void> updateServerMetrics();
 
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
@@ -180,4 +182,6 @@ private:
 	// Calculate an "average" of the metrics replies that we received.  Penalize teams from which we did not receive all
 	// replies.
 	int64_t getLoadAverage() const;
+
+	bool allServersHaveHealthyAvailableSpace() const;
 };

From 3fe6a952f145f8ca2331a44810131cb01e535d48 Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Wed, 16 Feb 2022 10:28:55 -0800
Subject: [PATCH 075/210] Merge with upstream tcinfo refactor and move the
 server knob init to be adjacent to related knobs

---
 fdbclient/ServerKnobs.cpp  | 2 +-
 fdbclient/ServerKnobs.h    | 2 +-
 fdbserver/TCInfo.actor.cpp | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index a24100870e..d507bb7dd7 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -233,6 +233,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( STORAGE_METRICS_POLLING_DELAY,                         2.0 ); if( randomize && BUGGIFY ) STORAGE_METRICS_POLLING_DELAY = 15.0;
 	init( STORAGE_METRICS_RANDOM_DELAY,                          0.2 );
 	init( AVAILABLE_SPACE_RATIO_CUTOFF,                         0.05 );
+	init( AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER,           0.03 );
 	init( DESIRED_TEAMS_PER_SERVER,                                5 ); if( randomize && BUGGIFY ) DESIRED_TEAMS_PER_SERVER = deterministicRandom()->randomInt(1, 10);
 	init( MAX_TEAMS_PER_SERVER,           5*DESIRED_TEAMS_PER_SERVER );
 	init( DD_SHARD_SIZE_GRANULARITY,                         5000000 );
@@ -579,7 +580,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( MIN_AVAILABLE_SPACE,                                   1e8 );
 	init( MIN_AVAILABLE_SPACE_RATIO,                            0.05 );
-	init( AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER,           0.03 );
 	init( TARGET_AVAILABLE_SPACE_RATIO,                         0.30 );
 	init( AVAILABLE_SPACE_UPDATE_DELAY,                          5.0 );
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 1eac4c27e2..9f442f3470 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -185,6 +185,7 @@ public:
 	double STORAGE_METRICS_POLLING_DELAY;
 	double STORAGE_METRICS_RANDOM_DELAY;
 	double AVAILABLE_SPACE_RATIO_CUTOFF;
+	double AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
 	int DESIRED_TEAMS_PER_SERVER;
 	int MAX_TEAMS_PER_SERVER;
 	int64_t DD_SHARD_SIZE_GRANULARITY;
@@ -525,7 +526,6 @@ public:
 
 	int64_t MIN_AVAILABLE_SPACE;
 	double MIN_AVAILABLE_SPACE_RATIO;
-	double AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
 	double TARGET_AVAILABLE_SPACE_RATIO;
 	double AVAILABLE_SPACE_UPDATE_DELAY;
 
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index 87212ead0f..e85a0aeb83 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -138,9 +138,9 @@ TCServerInfo::TCServerInfo(StorageServerInterface ssi,
 }
 
 bool TCServerInfo::hasHealthyAvailableSpace(double minAvailableSpaceRatio) const {
-	ASSERT(serverMetrics.present());
+	ASSERT(serverMetricsPresent());
 
-	auto& metrics = serverMetrics.get();
+	auto& metrics = getServerMetrics();
 	ASSERT(metrics.available.bytes >= 0);
 	ASSERT(metrics.capacity.bytes >= 0);
 
@@ -403,7 +403,7 @@ bool TCTeamInfo::allServersHaveHealthyAvailableSpace() const {
 	double minAvailableSpaceRatio =
 	    SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF + SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
 	for (const auto& server : servers) {
-		if (!server->serverMetrics.present() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) {
+		if (!server->serverMetricsPresent() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) {
 			result = false;
 			break;
 		}

From 3dfdc6943c37829c05ca454ed4a630e54bb93bfe Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Wed, 16 Feb 2022 10:48:31 -0800
Subject: [PATCH 076/210] move downgrade/upgrade test to XXX_7.1.0 because in
 7.0 the workload doesn't have the fix

---
 fdbserver/workloads/ConfigureDatabase.actor.cpp    | 14 ++++++++++++--
 tests/CMakeLists.txt                               | 10 +++++-----
 .../ConfigureStorageMigrationTestRestart-1.toml    |  1 +
 .../ConfigureStorageMigrationTestRestart-2.toml    |  0
 .../ConfigureStorageMigrationTestRestart-1.toml    |  6 ++++++
 .../ConfigureStorageMigrationTestRestart-2.toml    |  3 +++
 6 files changed, 27 insertions(+), 7 deletions(-)
 rename tests/restarting/{from_7.0.0 => from_7.1.0}/ConfigureStorageMigrationTestRestart-1.toml (90%)
 rename tests/restarting/{from_7.0.0 => from_7.1.0}/ConfigureStorageMigrationTestRestart-2.toml (100%)
 rename tests/restarting/{to_7.0.0 => to_7.1.0}/ConfigureStorageMigrationTestRestart-1.toml (75%)
 rename tests/restarting/{to_7.0.0 => to_7.1.0}/ConfigureStorageMigrationTestRestart-2.toml (78%)

diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp
index c9ea1f1895..c40f953802 100644
--- a/fdbserver/workloads/ConfigureDatabase.actor.cpp
+++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp
@@ -38,7 +38,9 @@ static const char* storageMigrationTypes[] = { "perpetual_storage_wiggle=0 stora
 	                                           "storage_migration_type=aggressive" };
 static const char* logTypes[] = { "log_engine:=1",  "log_engine:=2",  "log_spill:=1",   "log_spill:=2",
 	                              "log_version:=2", "log_version:=3", "log_version:=4", "log_version:=5",
-	                              "log_version:=6", "log_version:=7" };
+	                              "log_version:=6",
+	                              // downgrade incompatible log version
+	                              "log_version:=7" };
 static const char* redundancies[] = { "single", "double", "triple" };
 static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_enabled:=1" };
 
@@ -221,6 +223,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	bool allowDescriptorChange;
 	bool allowTestStorageMigration;
 	bool waitStoreTypeCheck;
+	bool downgradeTest1; // if this is true, don't pick up downgrade incompatible config
 	std::vector<Future<Void>> clients;
 	PerfIntCounter retries;
 
@@ -231,6 +234,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 		allowTestStorageMigration =
 		    getOption(options, "allowTestStorageMigration"_sr, false) && g_simulator.allowStorageMigrationTypeChange;
 		waitStoreTypeCheck = getOption(options, "waitStoreTypeCheck"_sr, false);
+		downgradeTest1 = getOption(options, "downgradeTest1"_sr, false);
 		g_simulator.usableRegions = 1;
 	}
 
@@ -406,8 +410,14 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				    true)));
 			} else if (randomChoice == 6) {
 				// Some configurations will be invalid, and that's fine.
+				int length = sizeof(logTypes) / sizeof(logTypes[0]);
+
+				if(self->downgradeTest1) {
+					length -= 1;
+				}
+
 				wait(success(IssueConfigurationChange(
-				    cx, logTypes[deterministicRandom()->randomInt(0, sizeof(logTypes) / sizeof(logTypes[0]))], false)));
+				    cx, logTypes[deterministicRandom()->randomInt(0, length)], false)));
 			} else if (randomChoice == 7) {
 				wait(success(IssueConfigurationChange(
 				    cx,
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6c52b440fa..9bf5ac7e34 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -243,9 +243,6 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/to_7.0.0/CycleTestRestart-1.txt
                restarting/to_7.0.0/CycleTestRestart-2.txt)
-  add_fdb_test(
-    TEST_FILES restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
-               restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml)
   add_fdb_test(
     TEST_FILES restarting/from_7.0.0/SnapTestAttrition-1.txt
                restarting/from_7.0.0/SnapTestAttrition-2.txt)
@@ -259,8 +256,11 @@ if(WITH_PYTHON)
     TEST_FILES restarting/from_7.0.0/SnapCycleRestart-1.txt
                restarting/from_7.0.0/SnapCycleRestart-2.txt)
   add_fdb_test(
-    TEST_FILES restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
-               restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml)
+    TEST_FILES restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
+               restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-2.toml)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
+               restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml)
 
 
   add_fdb_test(TEST_FILES slow/ApiCorrectness.toml)
diff --git a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
similarity index 90%
rename from tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
rename to tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
index a70a9e70bf..cc59b943f0 100644
--- a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
+++ b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
@@ -8,6 +8,7 @@ clearAfterTest = false
     testName = 'ConfigureDatabase'
     testDuration = 30.0
     allowTestStorageMigration = true
+    allowDescriptorChange = false
 
     [[test.workload]]
     testName = 'RandomClogging'
diff --git a/tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
similarity index 100%
rename from tests/restarting/from_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
rename to tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
diff --git a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
similarity index 75%
rename from tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
rename to tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
index a70a9e70bf..b5fdf679ab 100644
--- a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-1.toml
+++ b/tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
@@ -1,4 +1,8 @@
+[configuration]
 extraMachineCountDC = 2
+maxTLogVersion=6
+disableHostname=true
+storageEngineExcludeTypes=[4]
 
 [[test]]
 testTitle = 'CloggedConfigureDatabaseTest'
@@ -8,6 +12,8 @@ clearAfterTest = false
     testName = 'ConfigureDatabase'
     testDuration = 30.0
     allowTestStorageMigration = true
+    allowDescriptorChange = false
+    downgradeTest1 = true
 
     [[test.workload]]
     testName = 'RandomClogging'
diff --git a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
similarity index 78%
rename from tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
rename to tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
index f8361ab8c7..9fa2c4c784 100644
--- a/tests/restarting/to_7.0.0/ConfigureStorageMigrationTestRestart-2.toml
+++ b/tests/restarting/to_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
@@ -1,12 +1,15 @@
+[configuration]
 extraMachineCountDC = 2
 
 [[test]]
 testTitle = 'CloggedConfigureDatabaseTest'
 runSetup=false
+waitForQuiescenceBegin=false
 
     [[test.workload]]
     testName = 'ConfigureDatabase'
     testDuration = 300.0
+    waitStoreTypeCheck = true
 
     [[test.workload]]
     testName = 'RandomClogging'

From 949f1f1c3e0bac2fa32fd5058b52d741512c81e7 Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Wed, 16 Feb 2022 11:33:07 -0800
Subject: [PATCH 077/210] Switch to testing MIN_AVAILABLE_SPACE

---
 fdbclient/ServerKnobs.cpp  | 2 +-
 fdbclient/ServerKnobs.h    | 2 +-
 fdbserver/TCInfo.actor.cpp | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index d507bb7dd7..3c68f8c712 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -233,7 +233,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( STORAGE_METRICS_POLLING_DELAY,                         2.0 ); if( randomize && BUGGIFY ) STORAGE_METRICS_POLLING_DELAY = 15.0;
 	init( STORAGE_METRICS_RANDOM_DELAY,                          0.2 );
 	init( AVAILABLE_SPACE_RATIO_CUTOFF,                         0.05 );
-	init( AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER,           0.03 );
 	init( DESIRED_TEAMS_PER_SERVER,                                5 ); if( randomize && BUGGIFY ) DESIRED_TEAMS_PER_SERVER = deterministicRandom()->randomInt(1, 10);
 	init( MAX_TEAMS_PER_SERVER,           5*DESIRED_TEAMS_PER_SERVER );
 	init( DD_SHARD_SIZE_GRANULARITY,                         5000000 );
@@ -580,6 +579,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( MIN_AVAILABLE_SPACE,                                   1e8 );
 	init( MIN_AVAILABLE_SPACE_RATIO,                            0.05 );
+	init( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER,              0.03 );
 	init( TARGET_AVAILABLE_SPACE_RATIO,                         0.30 );
 	init( AVAILABLE_SPACE_UPDATE_DELAY,                          5.0 );
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 9f442f3470..5bb48fc3e2 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -185,7 +185,6 @@ public:
 	double STORAGE_METRICS_POLLING_DELAY;
 	double STORAGE_METRICS_RANDOM_DELAY;
 	double AVAILABLE_SPACE_RATIO_CUTOFF;
-	double AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
 	int DESIRED_TEAMS_PER_SERVER;
 	int MAX_TEAMS_PER_SERVER;
 	int64_t DD_SHARD_SIZE_GRANULARITY;
@@ -526,6 +525,7 @@ public:
 
 	int64_t MIN_AVAILABLE_SPACE;
 	double MIN_AVAILABLE_SPACE_RATIO;
+	double MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER;
 	double TARGET_AVAILABLE_SPACE_RATIO;
 	double AVAILABLE_SPACE_UPDATE_DELAY;
 
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index e85a0aeb83..5cf7f6b2dd 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -401,7 +401,7 @@ double TCTeamInfo::getMinAvailableSpaceRatio(bool includeInFlight) const {
 bool TCTeamInfo::allServersHaveHealthyAvailableSpace() const {
 	bool result = true;
 	double minAvailableSpaceRatio =
-	    SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF + SERVER_KNOBS->AVAILABLE_SPACE_RATIO_CUTOFF_SAFETY_BUFFER;
+	    SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO + SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER;
 	for (const auto& server : servers) {
 		if (!server->serverMetricsPresent() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) {
 			result = false;

From e3909ad05e3df3b19ffdfb2c3888a9c76c29faab Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Wed, 16 Feb 2022 11:54:31 -0800
Subject: [PATCH 078/210] modify test file

---
 .../from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml       | 1 +
 .../from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
index cc59b943f0..162b8c68a1 100644
--- a/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
+++ b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-1.toml
@@ -1,3 +1,4 @@
+[configuration]
 extraMachineCountDC = 2
 
 [[test]]
diff --git a/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
index aac15a614d..9fa2c4c784 100644
--- a/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
+++ b/tests/restarting/from_7.1.0/ConfigureStorageMigrationTestRestart-2.toml
@@ -1,3 +1,4 @@
+[configuration]
 extraMachineCountDC = 2
 
 [[test]]

From 5846dda41029628636a234102df220e9c49b9041 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 16 Feb 2022 15:05:25 -0500
Subject: [PATCH 079/210] temporary changes and extra traces for debugging

---
 fdbserver/GrvProxyServer.actor.cpp | 11 +++++++++--
 flow/Trace.cpp                     | 20 ++++++++++----------
 flow/Trace.h                       |  2 +-
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 4aae181e57..0cea07c447 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -656,6 +656,11 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			}
 		}
 
+		TraceEvent("DebugGrvProxyThrottleTimes")
+		    .detail("LastBatchThrottled", stats->lastBatchQueueThrottled)
+		    .detail("BatchThrottleStart", stats->batchThrottleStartTime)
+		    .detail("LastDefaultThrottled", stats->lastDefaultQueueThrottled)
+		    .detail("DefaultThrottleStart", stats->defaultThrottleStartTime);
 		if (stats->lastBatchQueueThrottled) {
 			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 			if (now() - stats->batchThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
@@ -871,17 +876,19 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			requestsToStart++;
 		}
 		if (!batchQueue.empty()) {
-			TraceEvent("DebugGrvProxyBatchQueueEmpty").detail("Size", batchQueue.size());
+			TraceEvent("DebugGrvProxyBatchQueueNonEmpty").detail("Size", batchQueue.size());
 			grvProxyData->stats.lastBatchQueueThrottled = true;
 			grvProxyData->stats.batchThrottleStartTime = now();
 		} else {
+			TraceEvent("DebugGrvProxyBatchQueueEmpty");
 			grvProxyData->stats.lastBatchQueueThrottled = false;
 		}
 		if (!defaultQueue.empty()) {
-			TraceEvent("DebugGrvProxyDefaultQueueEmpty").detail("Size", defaultQueue.size());
+			TraceEvent("DebugGrvProxyDefaultQueueNonEmpty").detail("Size", defaultQueue.size());
 			grvProxyData->stats.lastDefaultQueueThrottled = true;
 			grvProxyData->stats.defaultThrottleStartTime = now();
 		} else {
+			TraceEvent("DebugGrvProxyDefaultQueueEmpty");
 			grvProxyData->stats.lastDefaultQueueThrottled = false;
 		}
 
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 7aabc10a60..4aafd83397 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -927,16 +927,16 @@ bool TraceEvent::init() {
 	enabled = enabled && (!g_network || severity >= FLOW_KNOBS->MIN_TRACE_SEVERITY);
 
 	// Backstop to throttle very spammy trace events
-	if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
-		if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
-			enabled = false;
-			TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
-			    .suppressFor(5);
-		} else {
-			traceEventThrottlerCache->addAndExpire(
-			    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
-		}
-	}
+	// if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
+	// 	if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
+	// 		enabled = false;
+	// 		TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
+	// 		    .suppressFor(5);
+	// 	} else {
+	// 		traceEventThrottlerCache->addAndExpire(
+	// 		    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
+	// 	}
+	// }
 
 	if (enabled) {
 		tmpEventMetric = std::make_unique<DynamicEventMetric>(MetricNameRef());
diff --git a/flow/Trace.h b/flow/Trace.h
index b30b5405de..8779a16a13 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -33,7 +33,7 @@
 #include "flow/Error.h"
 #include "flow/ITrace.h"
 
-#define TRACE_DEFAULT_ROLL_SIZE (10 << 20)
+#define TRACE_DEFAULT_ROLL_SIZE (10 << 24)
 #define TRACE_DEFAULT_MAX_LOGS_SIZE (10 * TRACE_DEFAULT_ROLL_SIZE)
 
 inline int fastrand() {

From 6199faadc3f915abc7b9d010ea832d8432d130cc Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 16 Feb 2022 16:52:38 -0500
Subject: [PATCH 080/210] fix bug which constantly overwrote start time of last
 throttled queue

---
 fdbserver/GrvProxyServer.actor.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 0cea07c447..214d901242 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -877,16 +877,20 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 		}
 		if (!batchQueue.empty()) {
 			TraceEvent("DebugGrvProxyBatchQueueNonEmpty").detail("Size", batchQueue.size());
-			grvProxyData->stats.lastBatchQueueThrottled = true;
-			grvProxyData->stats.batchThrottleStartTime = now();
+			if (!grvProxyData->stats.lastBatchQueueThrottled) {
+				grvProxyData->stats.lastBatchQueueThrottled = true;
+				grvProxyData->stats.batchThrottleStartTime = now();
+			}
 		} else {
 			TraceEvent("DebugGrvProxyBatchQueueEmpty");
 			grvProxyData->stats.lastBatchQueueThrottled = false;
 		}
 		if (!defaultQueue.empty()) {
 			TraceEvent("DebugGrvProxyDefaultQueueNonEmpty").detail("Size", defaultQueue.size());
-			grvProxyData->stats.lastDefaultQueueThrottled = true;
-			grvProxyData->stats.defaultThrottleStartTime = now();
+			if (!grvProxyData->stats.lastDefaultQueueThrottled) {
+				grvProxyData->stats.lastDefaultQueueThrottled = true;
+				grvProxyData->stats.defaultThrottleStartTime = now();
+			}
 		} else {
 			TraceEvent("DebugGrvProxyDefaultQueueEmpty");
 			grvProxyData->stats.lastDefaultQueueThrottled = false;

From e0d3b0a4887bfea67d6af5c1e4bd8a4d92c98bb4 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 16 Feb 2022 16:55:14 -0500
Subject: [PATCH 081/210] format times in trace event

---
 fdbserver/GrvProxyServer.actor.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 214d901242..eeb97ee516 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -657,10 +657,10 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		}
 
 		TraceEvent("DebugGrvProxyThrottleTimes")
-		    .detail("LastBatchThrottled", stats->lastBatchQueueThrottled)
-		    .detail("BatchThrottleStart", stats->batchThrottleStartTime)
-		    .detail("LastDefaultThrottled", stats->lastDefaultQueueThrottled)
-		    .detail("DefaultThrottleStart", stats->defaultThrottleStartTime);
+		    .detail("LastBatchThrottled", format("%.6f", stats->lastBatchQueueThrottled))
+		    .detail("BatchThrottleStart", format("%.6f", stats->batchThrottleStartTime))
+		    .detail("LastDefaultThrottled", format("%.6f", stats->lastDefaultQueueThrottled))
+		    .detail("DefaultThrottleStart", format("%.6f", stats->defaultThrottleStartTime));
 		if (stats->lastBatchQueueThrottled) {
 			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 			if (now() - stats->batchThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {

From f2b59af9b07f04366bc2f75630b4439c0538256a Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Wed, 16 Feb 2022 16:32:06 -0800
Subject: [PATCH 082/210] format code

---
 fdbserver/workloads/ConfigureDatabase.actor.cpp | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp
index c40f953802..10832a9785 100644
--- a/fdbserver/workloads/ConfigureDatabase.actor.cpp
+++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp
@@ -36,8 +36,14 @@ static const char* storageMigrationTypes[] = { "perpetual_storage_wiggle=0 stora
 	                                           "perpetual_storage_wiggle=1",
 	                                           "perpetual_storage_wiggle=1 storage_migration_type=gradual",
 	                                           "storage_migration_type=aggressive" };
-static const char* logTypes[] = { "log_engine:=1",  "log_engine:=2",  "log_spill:=1",   "log_spill:=2",
-	                              "log_version:=2", "log_version:=3", "log_version:=4", "log_version:=5",
+static const char* logTypes[] = { "log_engine:=1",
+	                              "log_engine:=2",
+	                              "log_spill:=1",
+	                              "log_spill:=2",
+	                              "log_version:=2",
+	                              "log_version:=3",
+	                              "log_version:=4",
+	                              "log_version:=5",
 	                              "log_version:=6",
 	                              // downgrade incompatible log version
 	                              "log_version:=7" };
@@ -412,12 +418,12 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				// Some configurations will be invalid, and that's fine.
 				int length = sizeof(logTypes) / sizeof(logTypes[0]);
 
-				if(self->downgradeTest1) {
+				if (self->downgradeTest1) {
 					length -= 1;
 				}
 
-				wait(success(IssueConfigurationChange(
-				    cx, logTypes[deterministicRandom()->randomInt(0, length)], false)));
+				wait(success(
+				    IssueConfigurationChange(cx, logTypes[deterministicRandom()->randomInt(0, length)], false)));
 			} else if (randomChoice == 7) {
 				wait(success(IssueConfigurationChange(
 				    cx,

From 7a17dbeaccc65a82dd84a8b28dfe82200d91693a Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Wed, 16 Feb 2022 19:18:24 -0800
Subject: [PATCH 083/210] Create DD unit test class. Start to make
 test-specific methods in DDTeamCollection private

---
 fdbserver/DDTeamCollection.actor.cpp | 1119 ++++++++++++++------------
 fdbserver/DDTeamCollection.h         |   11 +-
 2 files changed, 596 insertions(+), 534 deletions(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index c6dc10b0f1..47b14bae6d 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -5089,593 +5089,656 @@ Future<Void> DDTeamCollection::printSnapshotTeamsInfo(Reference<DDTeamCollection
 	return DDTeamCollectionImpl::printSnapshotTeamsInfo(self);
 }
 
-std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
-                                                     Reference<IReplicationPolicy> policy,
-                                                     int processCount) {
-	Database database = DatabaseContext::create(
-	    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
+class DDTeamCollectionUnitTest {
+	static std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
+	                                                            Reference<IReplicationPolicy> policy,
+	                                                            int processCount) {
+		Database database = DatabaseContext::create(
+		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
 
-	DatabaseConfiguration conf;
-	conf.storageTeamSize = teamSize;
-	conf.storagePolicy = policy;
+		DatabaseConfiguration conf;
+		conf.storageTeamSize = teamSize;
+		conf.storagePolicy = policy;
 
-	auto collection =
-	    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
-	                                                           UID(0, 0),
-	                                                           MoveKeysLock(),
-	                                                           PromiseStream<RelocateShard>(),
-	                                                           makeReference<ShardsAffectedByTeamFailure>(),
-	                                                           conf,
-	                                                           {},
-	                                                           {},
-	                                                           Future<Void>(Void()),
-	                                                           makeReference<AsyncVar<bool>>(true),
-	                                                           IsPrimary::True,
-	                                                           makeReference<AsyncVar<bool>>(false),
-	                                                           makeReference<AsyncVar<bool>>(false),
-	                                                           PromiseStream<GetMetricsRequest>(),
-	                                                           Promise<UID>(),
-	                                                           PromiseStream<Promise<int>>()));
+		auto collection =
+		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
+		                                                           UID(0, 0),
+		                                                           MoveKeysLock(),
+		                                                           PromiseStream<RelocateShard>(),
+		                                                           makeReference<ShardsAffectedByTeamFailure>(),
+		                                                           conf,
+		                                                           {},
+		                                                           {},
+		                                                           Future<Void>(Void()),
+		                                                           makeReference<AsyncVar<bool>>(true),
+		                                                           IsPrimary::True,
+		                                                           makeReference<AsyncVar<bool>>(false),
+		                                                           makeReference<AsyncVar<bool>>(false),
+		                                                           PromiseStream<GetMetricsRequest>(),
+		                                                           Promise<UID>(),
+		                                                           PromiseStream<Promise<int>>()));
 
-	for (int id = 1; id <= processCount; ++id) {
-		UID uid(id, 0);
-		StorageServerInterface interface;
-		interface.uniqueID = uid;
-		interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(id)));
-		interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(id % 5)));
-		interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(id % 3)));
-		collection->server_info[uid] = makeReference<TCServerInfo>(
-		    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
-		collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
-		collection->checkAndCreateMachine(collection->server_info[uid]);
+		for (int id = 1; id <= processCount; ++id) {
+			UID uid(id, 0);
+			StorageServerInterface interface;
+			interface.uniqueID = uid;
+			interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(id)));
+			interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(id % 5)));
+			interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(id % 3)));
+			collection->server_info[uid] = makeReference<TCServerInfo>(
+			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
+			collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
+			collection->checkAndCreateMachine(collection->server_info[uid]);
+		}
+
+		return collection;
 	}
 
-	return collection;
-}
+	static std::unique_ptr<DDTeamCollection> testMachineTeamCollection(int teamSize,
+	                                                                   Reference<IReplicationPolicy> policy,
+	                                                                   int processCount) {
+		Database database = DatabaseContext::create(
+		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
 
-std::unique_ptr<DDTeamCollection> testMachineTeamCollection(int teamSize,
-                                                            Reference<IReplicationPolicy> policy,
-                                                            int processCount) {
-	Database database = DatabaseContext::create(
-	    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
+		DatabaseConfiguration conf;
+		conf.storageTeamSize = teamSize;
+		conf.storagePolicy = policy;
 
-	DatabaseConfiguration conf;
-	conf.storageTeamSize = teamSize;
-	conf.storagePolicy = policy;
+		auto collection =
+		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
+		                                                           UID(0, 0),
+		                                                           MoveKeysLock(),
+		                                                           PromiseStream<RelocateShard>(),
+		                                                           makeReference<ShardsAffectedByTeamFailure>(),
+		                                                           conf,
+		                                                           {},
+		                                                           {},
+		                                                           Future<Void>(Void()),
+		                                                           makeReference<AsyncVar<bool>>(true),
+		                                                           IsPrimary::True,
+		                                                           makeReference<AsyncVar<bool>>(false),
+		                                                           makeReference<AsyncVar<bool>>(false),
+		                                                           PromiseStream<GetMetricsRequest>(),
+		                                                           Promise<UID>(),
+		                                                           PromiseStream<Promise<int>>()));
 
-	auto collection =
-	    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
-	                                                           UID(0, 0),
-	                                                           MoveKeysLock(),
-	                                                           PromiseStream<RelocateShard>(),
-	                                                           makeReference<ShardsAffectedByTeamFailure>(),
-	                                                           conf,
-	                                                           {},
-	                                                           {},
-	                                                           Future<Void>(Void()),
-	                                                           makeReference<AsyncVar<bool>>(true),
-	                                                           IsPrimary::True,
-	                                                           makeReference<AsyncVar<bool>>(false),
-	                                                           makeReference<AsyncVar<bool>>(false),
-	                                                           PromiseStream<GetMetricsRequest>(),
-	                                                           Promise<UID>(),
-	                                                           PromiseStream<Promise<int>>()));
+		for (int id = 1; id <= processCount; id++) {
+			UID uid(id, 0);
+			StorageServerInterface interface;
+			interface.uniqueID = uid;
+			int process_id = id;
+			int dc_id = process_id / 1000;
+			int data_hall_id = process_id / 100;
+			int zone_id = process_id / 10;
+			int machine_id = process_id / 5;
 
-	for (int id = 1; id <= processCount; id++) {
-		UID uid(id, 0);
-		StorageServerInterface interface;
-		interface.uniqueID = uid;
-		int process_id = id;
-		int dc_id = process_id / 1000;
-		int data_hall_id = process_id / 100;
-		int zone_id = process_id / 10;
-		int machine_id = process_id / 5;
+			printf("testMachineTeamCollection: process_id:%d zone_id:%d machine_id:%d ip_addr:%s\n",
+			       process_id,
+			       zone_id,
+			       machine_id,
+			       interface.address().toString().c_str());
+			interface.locality.set(LiteralStringRef("processid"), Standalone<StringRef>(std::to_string(process_id)));
+			interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(machine_id)));
+			interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(zone_id)));
+			interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(data_hall_id)));
+			interface.locality.set(LiteralStringRef("dcid"), Standalone<StringRef>(std::to_string(dc_id)));
+			collection->server_info[uid] = makeReference<TCServerInfo>(
+			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
 
-		printf("testMachineTeamCollection: process_id:%d zone_id:%d machine_id:%d ip_addr:%s\n",
-		       process_id,
-		       zone_id,
-		       machine_id,
-		       interface.address().toString().c_str());
-		interface.locality.set(LiteralStringRef("processid"), Standalone<StringRef>(std::to_string(process_id)));
-		interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(machine_id)));
-		interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(zone_id)));
-		interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(data_hall_id)));
-		interface.locality.set(LiteralStringRef("dcid"), Standalone<StringRef>(std::to_string(dc_id)));
-		collection->server_info[uid] = makeReference<TCServerInfo>(
-		    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
+			collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
+		}
 
-		collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
+		int totalServerIndex = collection->constructMachinesFromServers();
+		printf("testMachineTeamCollection: construct machines for %d servers\n", totalServerIndex);
+
+		return collection;
 	}
 
-	int totalServerIndex = collection->constructMachinesFromServers();
-	printf("testMachineTeamCollection: construct machines for %d servers\n", totalServerIndex);
+public:
+	ACTOR static Future<Void> AddTeamsBestOf_UseMachineID() {
+		wait(Future<Void>(Void()));
 
-	return collection;
-}
+		int teamSize = 3; // replication size
+		int processSize = 60;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
+
+		collection->addTeamsBestOf(30, desiredTeams, maxTeams);
+
+		ASSERT(collection->sanityCheckTeams() == true);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> AddTeamsBestOf_NotUseMachineID() {
+		wait(Future<Void>(Void()));
+
+		int teamSize = 3; // replication size
+		int processSize = 60;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
+
+		if (collection == nullptr) {
+			fprintf(stderr, "collection is null\n");
+			return Void();
+		}
+
+		collection->addBestMachineTeams(30); // Create machine teams to help debug
+		collection->addTeamsBestOf(30, desiredTeams, maxTeams);
+		collection->sanityCheckTeams(); // Server team may happen to be on the same machine team, although unlikely
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> AddAllTeams_isExhaustive() {
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 10;
+		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
+
+		int result = collection->addTeamsBestOf(200, desiredTeams, maxTeams);
+
+		// The maximum number of available server teams without considering machine locality is 120
+		// The maximum number of available server teams with machine locality constraint is 120 - 40, because
+		// the 40 (5*4*2) server teams whose servers come from the same machine are invalid.
+		ASSERT(result == 80);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> AddAllTeams_withLimit() {
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 10;
+		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
+
+		int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
+
+		ASSERT(result >= 10);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> AddTeamsBestOf_SkippingBusyServers() {
+		wait(Future<Void>(Void()));
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 10;
+		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		state int teamSize = 3;
+		// state int targetTeamsPerServer = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (teamSize + 1) / 2;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true);
+
+		state int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams);
+
+		ASSERT(result >= 8);
+
+		for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
+			auto teamCount = process->second->teams.size();
+			ASSERT(teamCount >= 1);
+			// ASSERT(teamCount <= targetTeamsPerServer);
+		}
+
+		return Void();
+	}
+
+	// Due to the randomness in choosing the machine team and the server team from the machine team, it is possible that
+	// we may not find the remaining several (e.g., 1 or 2) available teams.
+	// It is hard to conclude what is the minimum number of  teams the addTeamsBestOf() should create in this situation.
+	ACTOR static Future<Void> AddTeamsBestOf_NotEnoughServers() {
+		wait(Future<Void>(Void()));
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true);
+
+		collection->addBestMachineTeams(10);
+		int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
+
+		if (collection->machineTeams.size() != 10 || result != 8) {
+			collection->traceAllInfo(true); // Debug message
+		}
+
+		// NOTE: Due to the pure randomness in selecting a machine for a machine team,
+		// we cannot guarantee that all machine teams are created.
+		// When we chnage the selectReplicas function to achieve such guarantee, we can enable the following ASSERT
+		ASSERT(collection->machineTeams.size() == 10); // Should create all machine teams
+
+		// We need to guarantee a server always have at least a team so that the server can participate in data
+		// distribution
+		for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
+			auto teamCount = process->second->teams.size();
+			ASSERT(teamCount >= 1);
+		}
+
+		// If we find all available teams, result will be 8 because we prebuild 2 teams
+		ASSERT(result == 8);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_NewServersNotNeeded() {
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply mid_avail;
+		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
+		mid_avail.available.bytes = 400 * 1024 * 1024;
+		mid_avail.load.bytes = 100 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 1000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
+
+		/*
+		 * Suppose  1, 2 and 3 are complete sources, i.e., they have all shards in
+		 * the key range being considered for movement. If the caller says that they
+		 * don't strictly need new servers and all of these servers are healthy,
+		 * maintain status quo.
+		 */
+
+		bool wantsNewServers = false;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = true;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		std::set<UID> expectedServers{ UID(1, 0), UID(2, 0), UID(3, 0) };
+		ASSERT(resTeam.first.present());
+		auto servers = resTeam.first.get()->getServerIDs();
+		const std::set<UID> selectedServers(servers.begin(), servers.end());
+		ASSERT(expectedServers == selectedServers);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_HealthyCompleteSource() {
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply mid_avail;
+		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
+		mid_avail.available.bytes = 400 * 1024 * 1024;
+		mid_avail.load.bytes = 100 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 1000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
+
+		/*
+		 * Suppose  1, 2, 3 and 4 are complete sources, i.e., they have all shards in
+		 * the key range being considered for movement. If the caller says that they don't
+		 * strictly need new servers but '1' is not healthy, see that the other team of
+		 * complete sources is selected.
+		 */
+
+		bool wantsNewServers = false;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = true;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0), UID(4, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
+		ASSERT(resTeam.first.present());
+		auto servers = resTeam.first.get()->getServerIDs();
+		const std::set<UID> selectedServers(servers.begin(), servers.end());
+
+		ASSERT(expectedServers == selectedServers);
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_TrueBestLeastUtilized() {
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply mid_avail;
+		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
+		mid_avail.available.bytes = 400 * 1024 * 1024;
+		mid_avail.load.bytes = 100 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 1000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		/*
+		 * Among server teams that have healthy space available, pick the team that is
+		 * least utilized, if the caller says they preferLowerUtilization.
+		 */
+
+		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
+
+		bool wantsNewServers = true;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = true;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
+		ASSERT(resTeam.first.present());
+		auto servers = resTeam.first.get()->getServerIDs();
+		const std::set<UID> selectedServers(servers.begin(), servers.end());
+		ASSERT(expectedServers == selectedServers);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_TrueBestMostUtilized() {
+
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply mid_avail;
+		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
+		mid_avail.available.bytes = 400 * 1024 * 1024;
+		mid_avail.load.bytes = 100 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 1000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
+
+		/*
+		 * Among server teams that have healthy space available, pick the team that is
+		 * most utilized, if the caller says they don't preferLowerUtilization.
+		 */
+
+		bool wantsNewServers = true;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = false;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		std::set<UID> expectedServers{ UID(1, 0), UID(2, 0), UID(3, 0) };
+		ASSERT(resTeam.first.present());
+		auto servers = resTeam.first.get()->getServerIDs();
+		const std::set<UID> selectedServers(servers.begin(), servers.end());
+		ASSERT(expectedServers == selectedServers);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_ServerUtilizationBelowCutoff() {
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply low_avail;
+		low_avail.capacity.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE * 20;
+		low_avail.available.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE / 2;
+		low_avail.load.bytes = 90 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 2000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		collection->server_info[UID(1, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = low_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = low_avail;
+		collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
+
+		/*
+		 * If the only available team is one where at least one server is low on
+		 * space, decline to pick that team. Every server must have some minimum
+		 * free space defined by the MIN_AVAILABLE_SPACE server knob.
+		 */
+
+		bool wantsNewServers = true;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = true;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		ASSERT(!resTeam.first.present());
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> GetTeam_ServerUtilizationNearCutoff() {
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+
+		GetStorageMetricsReply low_avail;
+		if (SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO > 0) {
+			/* Pick a capacity where MIN_AVAILABLE_SPACE_RATIO of the capacity would be higher than MIN_AVAILABLE_SPACE
+			 */
+			low_avail.capacity.bytes =
+			    SERVER_KNOBS->MIN_AVAILABLE_SPACE * (2 / SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO);
+		} else {
+			low_avail.capacity.bytes = 2000 * 1024 * 1024;
+		}
+		low_avail.available.bytes = (SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * 1.1) * low_avail.capacity.bytes;
+		low_avail.load.bytes = 90 * 1024 * 1024;
+
+		GetStorageMetricsReply high_avail;
+		high_avail.capacity.bytes = 2000 * 1024 * 1024;
+		high_avail.available.bytes = 800 * 1024 * 1024;
+		high_avail.load.bytes = 90 * 1024 * 1024;
+
+		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
+		collection->addTeam(std::set<UID>({ UID(3, 0), UID(4, 0), UID(5, 0) }), true);
+		collection->disableBuildingTeams();
+		collection->setCheckTeamDelay();
+
+		collection->server_info[UID(1, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(2, 0)]->serverMetrics = low_avail;
+		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(4, 0)]->serverMetrics = low_avail;
+		collection->server_info[UID(5, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
+
+		/*
+		 * If the only available team is one where all servers are low on space,
+		 * test that each server has at least MIN_AVAILABLE_SPACE_RATIO (server knob)
+		 * percentage points of capacity free before picking that team.
+		 */
+
+		bool wantsNewServers = true;
+		bool wantsTrueBest = true;
+		bool preferLowerUtilization = true;
+		bool teamMustHaveShards = false;
+		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
+
+		state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
+		req.completeSources = completeSources;
+
+		wait(collection->getTeam(req));
+
+		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
+
+		std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
+		ASSERT(resTeam.first.present());
+		auto servers = resTeam.first.get()->getServerIDs();
+		const std::set<UID> selectedServers(servers.begin(), servers.end());
+		ASSERT(expectedServers == selectedServers);
+
+		return Void();
+	}
+};
 
 TEST_CASE("DataDistribution/AddTeamsBestOf/UseMachineID") {
-	wait(Future<Void>(Void()));
-
-	int teamSize = 3; // replication size
-	int processSize = 60;
-	int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-
-	Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
-	    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
-
-	collection->addTeamsBestOf(30, desiredTeams, maxTeams);
-
-	ASSERT(collection->sanityCheckTeams() == true);
-
+	wait(DDTeamCollectionUnitTest::AddTeamsBestOf_UseMachineID());
 	return Void();
 }
 
 TEST_CASE("DataDistribution/AddTeamsBestOf/NotUseMachineID") {
-	wait(Future<Void>(Void()));
-
-	int teamSize = 3; // replication size
-	int processSize = 60;
-	int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-
-	Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
-	    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
-
-	if (collection == nullptr) {
-		fprintf(stderr, "collection is null\n");
-		return Void();
-	}
-
-	collection->addBestMachineTeams(30); // Create machine teams to help debug
-	collection->addTeamsBestOf(30, desiredTeams, maxTeams);
-	collection->sanityCheckTeams(); // Server team may happen to be on the same machine team, although unlikely
-
+	wait(DDTeamCollectionUnitTest::AddTeamsBestOf_NotUseMachineID());
 	return Void();
 }
 
 TEST_CASE("DataDistribution/AddAllTeams/isExhaustive") {
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 10;
-	state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
-
-	int result = collection->addTeamsBestOf(200, desiredTeams, maxTeams);
-
-	// The maximum number of available server teams without considering machine locality is 120
-	// The maximum number of available server teams with machine locality constraint is 120 - 40, because
-	// the 40 (5*4*2) server teams whose servers come from the same machine are invalid.
-	ASSERT(result == 80);
-
+	wait(DDTeamCollectionUnitTest::AddAllTeams_isExhaustive());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/AddAllTeams/withLimit") {
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 10;
-	state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
-
-	int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
-
-	ASSERT(result >= 10);
-
+	wait(DDTeamCollectionUnitTest::AddAllTeams_withLimit());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/AddTeamsBestOf/SkippingBusyServers") {
-	wait(Future<Void>(Void()));
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 10;
-	state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-	state int teamSize = 3;
-	// state int targetTeamsPerServer = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (teamSize + 1) / 2;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true);
-
-	state int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams);
-
-	ASSERT(result >= 8);
-
-	for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
-		auto teamCount = process->second->teams.size();
-		ASSERT(teamCount >= 1);
-		// ASSERT(teamCount <= targetTeamsPerServer);
-	}
-
+	wait(DDTeamCollectionUnitTest::AddTeamsBestOf_SkippingBusyServers());
 	return Void();
 }
 
-// Due to the randomness in choosing the machine team and the server team from the machine team, it is possible that
-// we may not find the remaining several (e.g., 1 or 2) available teams.
-// It is hard to conclude what is the minimum number of  teams the addTeamsBestOf() should create in this situation.
 TEST_CASE("/DataDistribution/AddTeamsBestOf/NotEnoughServers") {
-	wait(Future<Void>(Void()));
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-	state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true);
-
-	collection->addBestMachineTeams(10);
-	int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
-
-	if (collection->machineTeams.size() != 10 || result != 8) {
-		collection->traceAllInfo(true); // Debug message
-	}
-
-	// NOTE: Due to the pure randomness in selecting a machine for a machine team,
-	// we cannot guarantee that all machine teams are created.
-	// When we chnage the selectReplicas function to achieve such guarantee, we can enable the following ASSERT
-	ASSERT(collection->machineTeams.size() == 10); // Should create all machine teams
-
-	// We need to guarantee a server always have at least a team so that the server can participate in data distribution
-	for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
-		auto teamCount = process->second->teams.size();
-		ASSERT(teamCount >= 1);
-	}
-
-	// If we find all available teams, result will be 8 because we prebuild 2 teams
-	ASSERT(result == 8);
-
+	wait(DDTeamCollectionUnitTest::AddTeamsBestOf_NotEnoughServers());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/NewServersNotNeeded") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply mid_avail;
-	mid_avail.capacity.bytes = 1000 * 1024 * 1024;
-	mid_avail.available.bytes = 400 * 1024 * 1024;
-	mid_avail.load.bytes = 100 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 1000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
-
-	/*
-	 * Suppose  1, 2 and 3 are complete sources, i.e., they have all shards in
-	 * the key range being considered for movement. If the caller says that they
-	 * don't strictly need new servers and all of these servers are healthy,
-	 * maintain status quo.
-	 */
-
-	bool wantsNewServers = false;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = true;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	std::set<UID> expectedServers{ UID(1, 0), UID(2, 0), UID(3, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-	ASSERT(expectedServers == selectedServers);
-
+	wait(DDTeamCollectionUnitTest::GetTeam_NewServersNotNeeded());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/HealthyCompleteSource") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply mid_avail;
-	mid_avail.capacity.bytes = 1000 * 1024 * 1024;
-	mid_avail.available.bytes = 400 * 1024 * 1024;
-	mid_avail.load.bytes = 100 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 1000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
-
-	/*
-	 * Suppose  1, 2, 3 and 4 are complete sources, i.e., they have all shards in
-	 * the key range being considered for movement. If the caller says that they don't
-	 * strictly need new servers but '1' is not healthy, see that the other team of
-	 * complete sources is selected.
-	 */
-
-	bool wantsNewServers = false;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = true;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0), UID(4, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-
-	ASSERT(expectedServers == selectedServers);
+	wait(DDTeamCollectionUnitTest::GetTeam_HealthyCompleteSource());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/TrueBestLeastUtilized") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply mid_avail;
-	mid_avail.capacity.bytes = 1000 * 1024 * 1024;
-	mid_avail.available.bytes = 400 * 1024 * 1024;
-	mid_avail.load.bytes = 100 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 1000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	/*
-	 * Among server teams that have healthy space available, pick the team that is
-	 * least utilized, if the caller says they preferLowerUtilization.
-	 */
-
-	collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
-
-	bool wantsNewServers = true;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = true;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-	ASSERT(expectedServers == selectedServers);
-
+	wait(DDTeamCollectionUnitTest::GetTeam_TrueBestLeastUtilized());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/TrueBestMostUtilized") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply mid_avail;
-	mid_avail.capacity.bytes = 1000 * 1024 * 1024;
-	mid_avail.available.bytes = 400 * 1024 * 1024;
-	mid_avail.load.bytes = 100 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 1000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
-
-	/*
-	 * Among server teams that have healthy space available, pick the team that is
-	 * most utilized, if the caller says they don't preferLowerUtilization.
-	 */
-
-	bool wantsNewServers = true;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = false;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	std::set<UID> expectedServers{ UID(1, 0), UID(2, 0), UID(3, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-	ASSERT(expectedServers == selectedServers);
-
+	wait(DDTeamCollectionUnitTest::GetTeam_TrueBestMostUtilized());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/ServerUtilizationBelowCutoff") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply low_avail;
-	low_avail.capacity.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE * 20;
-	low_avail.available.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE / 2;
-	low_avail.load.bytes = 90 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 2000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	collection->server_info[UID(1, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = low_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = low_avail;
-	collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
-
-	/*
-	 * If the only available team is one where at least one server is low on
-	 * space, decline to pick that team. Every server must have some minimum
-	 * free space defined by the MIN_AVAILABLE_SPACE server knob.
-	 */
-
-	bool wantsNewServers = true;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = true;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	ASSERT(!resTeam.first.present());
-
+	wait(DDTeamCollectionUnitTest::GetTeam_ServerUtilizationBelowCutoff());
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/GetTeam/ServerUtilizationNearCutoff") {
-
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply low_avail;
-	if (SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO > 0) {
-		/* Pick a capacity where MIN_AVAILABLE_SPACE_RATIO of the capacity would be higher than MIN_AVAILABLE_SPACE */
-		low_avail.capacity.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE * (2 / SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO);
-	} else {
-		low_avail.capacity.bytes = 2000 * 1024 * 1024;
-	}
-	low_avail.available.bytes = (SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * 1.1) * low_avail.capacity.bytes;
-	low_avail.load.bytes = 90 * 1024 * 1024;
-
-	GetStorageMetricsReply high_avail;
-	high_avail.capacity.bytes = 2000 * 1024 * 1024;
-	high_avail.available.bytes = 800 * 1024 * 1024;
-	high_avail.load.bytes = 90 * 1024 * 1024;
-
-	collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true);
-	collection->addTeam(std::set<UID>({ UID(3, 0), UID(4, 0), UID(5, 0) }), true);
-	collection->disableBuildingTeams();
-	collection->setCheckTeamDelay();
-
-	collection->server_info[UID(1, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(2, 0)]->serverMetrics = low_avail;
-	collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(4, 0)]->serverMetrics = low_avail;
-	collection->server_info[UID(5, 0)]->serverMetrics = high_avail;
-	collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
-
-	/*
-	 * If the only available team is one where all servers are low on space,
-	 * test that each server has at least MIN_AVAILABLE_SPACE_RATIO (server knob)
-	 * percentage points of capacity free before picking that team.
-	 */
-
-	bool wantsNewServers = true;
-	bool wantsTrueBest = true;
-	bool preferLowerUtilization = true;
-	bool teamMustHaveShards = false;
-	std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
-
-	state GetTeamRequest req(wantsNewServers, wantsTrueBest, preferLowerUtilization, teamMustHaveShards);
-	req.completeSources = completeSources;
-
-	wait(collection->getTeam(req));
-
-	std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
-
-	std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
-	ASSERT(resTeam.first.present());
-	auto servers = resTeam.first.get()->getServerIDs();
-	const std::set<UID> selectedServers(servers.begin(), servers.end());
-	ASSERT(expectedServers == selectedServers);
-
+	wait(DDTeamCollectionUnitTest::GetTeam_ServerUtilizationNearCutoff());
 	return Void();
 }
diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h
index ed4d27d248..3fa923ddff 100644
--- a/fdbserver/DDTeamCollection.h
+++ b/fdbserver/DDTeamCollection.h
@@ -171,6 +171,7 @@ typedef AsyncMap<UID, ServerStatus> ServerStatusMap;
 
 class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	friend class DDTeamCollectionImpl;
+	friend class DDTeamCollectionUnitTest;
 
 	enum class Status { NONE = 0, WIGGLING = 1, EXCLUDED = 2, FAILED = 3 };
 
@@ -521,6 +522,10 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 
 	void noHealthyTeams() const;
 
+	void disableBuildingTeams() { doBuildTeams = false; }
+
+	void setCheckTeamDelay() { this->checkTeamDelay = Void(); }
+
 public:
 	Database cx;
 
@@ -595,12 +600,6 @@ public:
 
 	void addTeam(std::set<UID> const& team, bool isInitialTeam) { addTeam(team.begin(), team.end(), isInitialTeam); }
 
-	// FIXME: Public for testing only
-	void disableBuildingTeams() { doBuildTeams = false; }
-
-	// FIXME: Public for testing only
-	void setCheckTeamDelay() { this->checkTeamDelay = Void(); }
-
 	// FIXME: Public for testing only
 	// Group storage servers (process) based on their machineId in LocalityData
 	// All created machines are healthy

From a54acb372004d5f5d368c2591b49e579573e56c3 Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Wed, 16 Feb 2022 19:26:40 -0800
Subject: [PATCH 084/210] Temporarily lower safety buffer knob.
 AtomicBackupCorrectness needs fixing

---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 1b218fdf9c..21f1b38390 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -581,7 +581,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( MIN_AVAILABLE_SPACE,                                   1e8 );
 	init( MIN_AVAILABLE_SPACE_RATIO,                            0.05 );
-	init( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER,              0.03 );
+	init( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER,              0.01 );
 	init( TARGET_AVAILABLE_SPACE_RATIO,                         0.30 );
 	init( AVAILABLE_SPACE_UPDATE_DELAY,                          5.0 );
 

From c568f9884ddb93308eef07be483c7239bf2e97bc Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Thu, 17 Feb 2022 08:25:20 -0800
Subject: [PATCH 085/210] Make a few more DDCollection methods private

---
 fdbserver/DDTeamCollection.h | 59 +++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 32 deletions(-)

diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h
index 3fa923ddff..86cd92e4cf 100644
--- a/fdbserver/DDTeamCollection.h
+++ b/fdbserver/DDTeamCollection.h
@@ -522,6 +522,33 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 
 	void noHealthyTeams() const;
 
+	// To enable verbose debug info, set shouldPrint to true
+	void traceAllInfo(bool shouldPrint = false) const;
+
+	// Check if the server belongs to a machine; if not, create the machine.
+	// Establish the two-direction link between server and machine
+	Reference<TCMachineInfo> checkAndCreateMachine(Reference<TCServerInfo> server);
+
+	// Group storage servers (process) based on their machineId in LocalityData
+	// All created machines are healthy
+	// Return The number of healthy servers we grouped into machines
+	int constructMachinesFromServers();
+
+	// Create machineTeamsToBuild number of machine teams
+	// No operation if machineTeamsToBuild is 0
+	// Note: The creation of machine teams should not depend on server teams:
+	// No matter how server teams will be created, we will create the same set of machine teams;
+	// We should never use server team number in building machine teams.
+	//
+	// Five steps to create each machine team, which are document in the function
+	// Reuse ReplicationPolicy selectReplicas func to select machine team
+	// return number of added machine teams
+	int addBestMachineTeams(int machineTeamsToBuild);
+
+	// Sanity check the property of teams in unit test
+	// Return true if all server teams belong to machine teams
+	bool sanityCheckTeams() const;
+
 	void disableBuildingTeams() { doBuildTeams = false; }
 
 	void setCheckTeamDelay() { this->checkTeamDelay = Void(); }
@@ -600,33 +627,6 @@ public:
 
 	void addTeam(std::set<UID> const& team, bool isInitialTeam) { addTeam(team.begin(), team.end(), isInitialTeam); }
 
-	// FIXME: Public for testing only
-	// Group storage servers (process) based on their machineId in LocalityData
-	// All created machines are healthy
-	// Return The number of healthy servers we grouped into machines
-	int constructMachinesFromServers();
-
-	// FIXME: Public for testing only
-	// To enable verbose debug info, set shouldPrint to true
-	void traceAllInfo(bool shouldPrint = false) const;
-
-	// FIXME: Public for testing only
-	// Create machineTeamsToBuild number of machine teams
-	// No operation if machineTeamsToBuild is 0
-	// Note: The creation of machine teams should not depend on server teams:
-	// No matter how server teams will be created, we will create the same set of machine teams;
-	// We should never use server team number in building machine teams.
-	//
-	// Five steps to create each machine team, which are document in the function
-	// Reuse ReplicationPolicy selectReplicas func to select machine team
-	// return number of added machine teams
-	int addBestMachineTeams(int machineTeamsToBuild);
-
-	// FIXME: Public for testing only
-	// Sanity check the property of teams in unit test
-	// Return true if all server teams belong to machine teams
-	bool sanityCheckTeams() const;
-
 	// Create server teams based on machine teams
 	// Before the number of machine teams reaches the threshold, build a machine team for each server team
 	// When it reaches the threshold, first try to build a server team with existing machine teams; if failed,
@@ -641,11 +641,6 @@ public:
 
 	bool removeTeam(Reference<TCTeamInfo> team);
 
-	// FIXME: Public for testing only
-	// Check if the server belongs to a machine; if not, create the machine.
-	// Establish the two-direction link between server and machine
-	Reference<TCMachineInfo> checkAndCreateMachine(Reference<TCServerInfo> server);
-
 	void removeTSS(UID removedServer);
 
 	void removeServer(UID removedServer);

From d01f035848b5f7d859134e4110e1285aa104daaa Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Thu, 17 Feb 2022 13:45:17 -0800
Subject: [PATCH 086/210] Increase ITERATIONS_PER_REACTOR_CHECK default

---
 flow/Knobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 496bde4e46..6f26ec3662 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -172,7 +172,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( MIN_LOGGED_PRIORITY_BUSY_FRACTION,                  0.05 );
 	init( CERT_FILE_MAX_SIZE,                      5 * 1024 * 1024 );
 	init( READY_QUEUE_RESERVED_SIZE,                          8192 );
-	init( ITERATIONS_PER_REACTOR_CHECK,                          5 );
+	init( ITERATIONS_PER_REACTOR_CHECK,                        100 );
 
 	//Network
 	init( PACKET_LIMIT,                                  100LL<<20 );

From c1b0b336ce0d1fc77302170f8da64f846511445e Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Thu, 17 Feb 2022 20:41:38 -0800
Subject: [PATCH 087/210] Fix merge conflicts in DDTC.actor.cpp - change
 serverMetrics to setServerMetrics and setTeamHealthy to markTeamUnhealthy

---
 fdbserver/DDTeamCollection.actor.cpp | 88 ++++++++++++++--------------
 1 file changed, 43 insertions(+), 45 deletions(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 231f629a24..fa564ea040 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -5261,13 +5261,13 @@ public:
 
 		ASSERT(result >= 8);
 
-	  for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
-		  auto teamCount = process->second->getTeams().size();
-  		ASSERT(teamCount >= 1);
-	  	// ASSERT(teamCount <= targetTeamsPerServer);
-    }
-  
-    return Void();
+		for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
+			auto teamCount = process->second->getTeams().size();
+			ASSERT(teamCount >= 1);
+			// ASSERT(teamCount <= targetTeamsPerServer);
+		}
+
+		return Void();
 	}
 
 	// Due to the randomness in choosing the machine team and the server team from the machine team, it is possible that
@@ -5302,7 +5302,7 @@ public:
 		// We need to guarantee a server always have at least a team so that the server can participate in data
 		// distribution
 		for (auto process = collection->server_info.begin(); process != collection->server_info.end(); process++) {
-			auto teamCount = process->second->getTeams.size();
+			auto teamCount = process->second->getTeams().size();
 			ASSERT(teamCount >= 1);
 		}
 
@@ -5335,10 +5335,10 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->setServerMetrics(mid_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(high_avail);
 
 		/*
 		 * Suppose  1, 2 and 3 are complete sources, i.e., they have all shards in
@@ -5391,11 +5391,11 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		collection->server_info[UID(1, 0)]->serverMetrics = mid_avail;
-		collection->server_info[UID(2, 0)]->serverMetrics = high_avail;
-		collection->server_info[UID(3, 0)]->serverMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->serverMetrics = high_avail;
-		collection->server_info[UID(1, 0)]->teams[0]->setHealthy(false);
+		collection->server_info[UID(1, 0)]->setServerMetrics(mid_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(1, 0)]->markTeamUnhealthy(0);
 
 		/*
 		 * Suppose  1, 2, 3 and 4 are complete sources, i.e., they have all shards in
@@ -5428,17 +5428,16 @@ public:
 
 	ACTOR static Future<Void> GetTeam_TrueBestLeastUtilized() {
 
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-	state int processSize = 5;
-	state int teamSize = 3;
-	state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-
-	GetStorageMetricsReply mid_avail;
-	mid_avail.capacity.bytes = 1000 * 1024 * 1024;
-	mid_avail.available.bytes = 400 * 1024 * 1024;
-	mid_avail.load.bytes = 100 * 1024 * 1024;
+		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
+		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		state int processSize = 5;
+		state int teamSize = 3;
+		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
+		GetStorageMetricsReply mid_avail;
+		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
+		mid_avail.available.bytes = 400 * 1024 * 1024;
+		mid_avail.load.bytes = 100 * 1024 * 1024;
 
 		GetStorageMetricsReply high_avail;
 		high_avail.capacity.bytes = 1000 * 1024 * 1024;
@@ -5450,16 +5449,15 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-
 		/*
 		 * Among server teams that have healthy space available, pick the team that is
 		 * least utilized, if the caller says they preferLowerUtilization.
 		 */
 
-		collection->server_info[UID(1, 0)]->setServerMetrics = mid_avail;
-		collection->server_info[UID(2, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(3, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->setServerMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->setServerMetrics(mid_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(high_avail);
 
 		bool wantsNewServers = true;
 		bool wantsTrueBest = true;
@@ -5506,10 +5504,10 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		collection->server_info[UID(1, 0)]->setServerMetrics = mid_avail;
-		collection->server_info[UID(2, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(3, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->setServerMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->setServerMetrics(mid_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(high_avail);
 
 		/*
 		 * Among server teams that have healthy space available, pick the team that is
@@ -5560,10 +5558,10 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		collection->server_info[UID(1, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(2, 0)]->setServerMetrics = low_avail;
-		collection->server_info[UID(3, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->setServerMetrics = low_avail;
+		collection->server_info[UID(1, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(low_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(low_avail);
 		collection->server_info[UID(1, 0)]->markTeamUnhealthy(0);
 
 		/*
@@ -5620,11 +5618,11 @@ public:
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		collection->server_info[UID(1, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(2, 0)]->setServerMetrics = low_avail;
-		collection->server_info[UID(3, 0)]->setServerMetrics = high_avail;
-		collection->server_info[UID(4, 0)]->setServerMetrics = low_avail;
-		collection->server_info[UID(5, 0)]->setServerMetrics = high_avail;
+		collection->server_info[UID(1, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(2, 0)]->setServerMetrics(low_avail);
+		collection->server_info[UID(3, 0)]->setServerMetrics(high_avail);
+		collection->server_info[UID(4, 0)]->setServerMetrics(low_avail);
+		collection->server_info[UID(5, 0)]->setServerMetrics(high_avail);
 		collection->server_info[UID(1, 0)]->markTeamUnhealthy(0);
 
 		/*

From 7bd767f1987329388191bcf9b539848efbd38692 Mon Sep 17 00:00:00 2001
From: vikasgupta8 <vikas.gupta8@ibm.com>
Date: Fri, 18 Feb 2022 09:35:45 +0000
Subject: [PATCH 088/210] Updated cmake process string to match with ppc64le or
 powerpc64le

---
 bindings/c/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 704e7a7dc0..f723e6311b 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -18,7 +18,7 @@ endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
   set(cpu "aarch64")
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "^(ppc64le|powerpc64le)")
   set(cpu "ppc64le")
 endif()
 

From d32d18ea643d0060c8e95497adacb5d3dbeaa772 Mon Sep 17 00:00:00 2001
From: Mohamed Oulmahdi <oulmahdi@doxense.com>
Date: Fri, 18 Feb 2022 11:46:09 +0100
Subject: [PATCH 089/210] Fix purgeOutdatedHistory

---
 fdbrpc/HealthMonitor.actor.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbrpc/HealthMonitor.actor.cpp b/fdbrpc/HealthMonitor.actor.cpp
index 772c3b7a9c..4ea1712e8b 100644
--- a/fdbrpc/HealthMonitor.actor.cpp
+++ b/fdbrpc/HealthMonitor.actor.cpp
@@ -29,12 +29,12 @@ void HealthMonitor::reportPeerClosed(const NetworkAddress& peerAddress) {
 }
 
 void HealthMonitor::purgeOutdatedHistory() {
-	for (auto it = peerClosedHistory.begin(); it != peerClosedHistory.end();) {
-		if (it->first < now() - FLOW_KNOBS->HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS) {
-			auto& count = peerClosedNum[it->second];
+	while (!peerClosedHistory.empty()) {
+		auto const& p = peerClosedHistory.front();
+		if (p.first < now() - FLOW_KNOBS->HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS) {
+			auto& count = peerClosedNum[p.second];
 			--count;
 			ASSERT(count >= 0);
-			++it; // Increment before pop_front to avoid iterator invalidation
 			peerClosedHistory.pop_front();
 		} else {
 			break;

From 7c7182166696b566e2bf15e8b5d3c55cbf623db9 Mon Sep 17 00:00:00 2001
From: vikasgupta8 <vikas.gupta8@ibm.com>
Date: Fri, 18 Feb 2022 12:53:23 +0000
Subject: [PATCH 090/210] Updated cmake process string to match with ppc64le or
 powerpc64le

---
 bindings/c/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index f723e6311b..8ba8e217fc 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -18,7 +18,7 @@ endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
   set(cpu "aarch64")
-elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "^(ppc64le|powerpc64le)")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le|powerpc64le)")
   set(cpu "ppc64le")
 endif()
 

From 6d101732e18d9117e62201f08a7130f56decc9e9 Mon Sep 17 00:00:00 2001
From: pluveto <50045289+pluveto@users.noreply.github.com>
Date: Mon, 21 Feb 2022 13:57:11 +0800
Subject: [PATCH 091/210] Fix invalid link to download page

Not available: https://www.foundationdb.org/download/
Change to https://github.com/apple/foundationdb/releases
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0404d9edab..fbcd4d3ef6 100755
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Contributing to FoundationDB can be in contributions to the code base, sharing y
 
 ### Binary downloads
 
-Developers interested in using FoundationDB can get started by downloading and installing a binary package. Please see the [downloads page](https://www.foundationdb.org/download/) for a list of available packages.
+Developers interested in using FoundationDB can get started by downloading and installing a binary package. Please see the [downloads page](https://github.com/apple/foundationdb/releases) for a list of available packages.
 
 
 ### Compiling from source
@@ -181,4 +181,4 @@ Under Windows, only Visual Studio with ClangCl is supported
 1. `mkdir build && cd build`
 1. `cmake -G "Visual Studio 16 2019" -A x64 -T ClangCl <PATH_TO_FOUNDATIONDB_SOURCE>`
 1. `msbuild /p:Configuration=Release foundationdb.sln`
-1. To increase build performance, use `/p:UseMultiToolTask=true` and `/p:CL_MPCount=<NUMBER_OF_PARALLEL_JOBS>` 
\ No newline at end of file
+1. To increase build performance, use `/p:UseMultiToolTask=true` and `/p:CL_MPCount=<NUMBER_OF_PARALLEL_JOBS>` 

From c29c522d3efff99a7fb265e3527c829d6a986506 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Fri, 18 Feb 2022 14:30:53 -0800
Subject: [PATCH 092/210] Change ClusterConnectionString's status flag from
 boolean to enum.

So that when multiple threads try to resolve hostnames concurrently, we should have only one resolving and others wait on the result. Otherwise, they would add duplicated resolved results to coordinators.
---
 fdbclient/CoordinationInterface.h |  22 ++++++-
 fdbclient/MonitorLeader.actor.cpp | 101 ++++++++++++++++++------------
 2 files changed, 79 insertions(+), 44 deletions(-)

diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index 9cf938931b..50860db0da 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -58,13 +58,28 @@ struct ClientLeaderRegInterface {
 //  - There is no address present more than once
 class ClusterConnectionString {
 public:
+	enum ConnectionStringStatus { RESOLVED, RESOLVING, UNRESOLVED };
+
 	ClusterConnectionString() {}
 	ClusterConnectionString(const std::string& connStr);
 	ClusterConnectionString(const std::vector<NetworkAddress>& coordinators, Key key);
 	ClusterConnectionString(const std::vector<Hostname>& hosts, Key key);
 
+	ClusterConnectionString(const ClusterConnectionString& rhs) { operator=(rhs); }
+	ClusterConnectionString& operator=(const ClusterConnectionString& rhs) {
+		// Copy everything except AsyncTrigger resolveFinish.
+		status = rhs.status;
+		coords = rhs.coords;
+		hostnames = rhs.hostnames;
+		networkAddressToHostname = rhs.networkAddressToHostname;
+		key = rhs.key;
+		keyDesc = rhs.keyDesc;
+		connectionString = rhs.connectionString;
+		return *this;
+	}
+
 	std::vector<NetworkAddress> const& coordinators() const { return coords; }
-	void addResolved(Hostname hostname, NetworkAddress address) {
+	void addResolved(const Hostname& hostname, const NetworkAddress& address) {
 		coords.push_back(address);
 		networkAddressToHostname.emplace(address, hostname);
 	}
@@ -80,7 +95,8 @@ public:
 	void resolveHostnamesBlocking();
 	void resetToUnresolved();
 
-	bool hasUnresolvedHostnames = false;
+	ConnectionStringStatus status = RESOLVED;
+	AsyncTrigger resolveFinish;
 	std::vector<NetworkAddress> coords;
 	std::vector<Hostname> hostnames;
 
@@ -139,7 +155,7 @@ public:
 	// Signals to the connection record that it was successfully used to connect to a cluster.
 	void notifyConnected();
 
-	bool hasUnresolvedHostnames() const;
+	ClusterConnectionString::ConnectionStringStatus connectionStringStatus() const;
 	Future<Void> resolveHostnames();
 	// This one should only be used when resolving asynchronously is impossible. For all other cases, resolveHostnames()
 	// should be preferred.
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 96c82013d1..4f237e1039 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -77,8 +77,8 @@ void IClusterConnectionRecord::setPersisted() {
 	connectionStringNeedsPersisted = false;
 }
 
-bool IClusterConnectionRecord::hasUnresolvedHostnames() const {
-	return cs.hasUnresolvedHostnames;
+ClusterConnectionString::ConnectionStringStatus IClusterConnectionRecord::connectionStringStatus() const {
+	return cs.status;
 }
 
 Future<Void> IClusterConnectionRecord::resolveHostnames() {
@@ -98,39 +98,56 @@ std::string ClusterConnectionString::getErrorString(std::string const& source, E
 }
 
 ACTOR Future<Void> resolveHostnamesImpl(ClusterConnectionString* self) {
-	std::vector<Future<Void>> fs;
-	for (auto const& hostName : self->hostnames) {
-		fs.push_back(map(INetworkConnections::net()->resolveTCPEndpoint(hostName.host, hostName.service),
-		                 [=](std::vector<NetworkAddress> const& addresses) -> Void {
-			                 NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
-			                 addr.flags = 0; // Reset the parsed address to public
-			                 addr.fromHostname = NetworkAddressFromHostname::True;
-			                 if (hostName.isTLS) {
-				                 addr.flags |= NetworkAddress::FLAG_TLS;
-			                 }
-			                 self->addResolved(hostName, addr);
-			                 return Void();
-		                 }));
+	loop {
+		if (self->status == ClusterConnectionString::UNRESOLVED) {
+			self->status = ClusterConnectionString::RESOLVING;
+			std::vector<Future<Void>> fs;
+			for (auto const& hostname : self->hostnames) {
+				fs.push_back(map(INetworkConnections::net()->resolveTCPEndpoint(hostname.host, hostname.service),
+				                 [=](std::vector<NetworkAddress> const& addresses) -> Void {
+					                 NetworkAddress address =
+					                     addresses[deterministicRandom()->randomInt(0, addresses.size())];
+					                 address.flags = 0; // Reset the parsed address to public
+					                 address.fromHostname = NetworkAddressFromHostname::True;
+					                 if (hostname.isTLS) {
+						                 address.flags |= NetworkAddress::FLAG_TLS;
+					                 }
+					                 self->addResolved(hostname, address);
+					                 return Void();
+				                 }));
+			}
+			wait(waitForAll(fs));
+			std::sort(self->coords.begin(), self->coords.end());
+			if (std::unique(self->coords.begin(), self->coords.end()) != self->coords.end()) {
+				self->status = ClusterConnectionString::UNRESOLVED;
+				self->resolveFinish.trigger();
+				throw connection_string_invalid();
+			}
+			self->status = ClusterConnectionString::RESOLVED;
+			self->resolveFinish.trigger();
+			break;
+		} else if (self->status == ClusterConnectionString::RESOLVING) {
+			wait(self->resolveFinish.onTrigger());
+			if (self->status == ClusterConnectionString::RESOLVED) {
+				break;
+			}
+			// Otherwise, this means other threads failed on resolve, so here we go back to the loop and try to resolve
+			// again.
+		} else {
+			// status is RESOLVED, nothing to do.
+			break;
+		}
 	}
-	wait(waitForAll(fs));
-	std::sort(self->coords.begin(), self->coords.end());
-	if (std::unique(self->coords.begin(), self->coords.end()) != self->coords.end()) {
-		throw connection_string_invalid();
-	}
-	self->hasUnresolvedHostnames = false;
 	return Void();
 }
 
 Future<Void> ClusterConnectionString::resolveHostnames() {
-	if (!hasUnresolvedHostnames) {
-		return Void();
-	} else {
-		return resolveHostnamesImpl(this);
-	}
+	return resolveHostnamesImpl(this);
 }
 
 void ClusterConnectionString::resolveHostnamesBlocking() {
-	if (hasUnresolvedHostnames) {
+	if (status != RESOLVED) {
+		status = RESOLVING;
 		for (auto const& hostname : hostnames) {
 			std::vector<NetworkAddress> addresses =
 			    INetworkConnections::net()->resolveTCPEndpointBlocking(hostname.host, hostname.service);
@@ -140,14 +157,14 @@ void ClusterConnectionString::resolveHostnamesBlocking() {
 			if (hostname.isTLS) {
 				address.flags |= NetworkAddress::FLAG_TLS;
 			}
-			coords.push_back(address);
-			networkAddressToHostname.emplace(address, hostname);
+			addResolved(hostname, address);
 		}
 		std::sort(coords.begin(), coords.end());
 		if (std::unique(coords.begin(), coords.end()) != coords.end()) {
+			status = UNRESOLVED;
 			throw connection_string_invalid();
 		}
-		hasUnresolvedHostnames = false;
+		status = RESOLVED;
 	}
 }
 
@@ -156,7 +173,7 @@ void ClusterConnectionString::resetToUnresolved() {
 		coords.clear();
 		hostnames.clear();
 		networkAddressToHostname.clear();
-		hasUnresolvedHostnames = true;
+		status = UNRESOLVED;
 		parseConnString();
 	}
 }
@@ -184,7 +201,9 @@ void ClusterConnectionString::parseConnString() {
 		}
 		p = pComma + 1;
 	}
-	hasUnresolvedHostnames = hostnames.size() > 0;
+	if (hostnames.size() > 0) {
+		status = UNRESOLVED;
+	}
 	ASSERT((coords.size() + hostnames.size()) > 0);
 
 	std::sort(coords.begin(), coords.end());
@@ -256,7 +275,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/hostnames") {
 	{
 		input = "asdf:2345@localhost:1234";
 		ClusterConnectionString cs(input);
-		ASSERT(cs.hasUnresolvedHostnames);
+		ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 		ASSERT(cs.hostnames.size() == 1);
 		ASSERT(input == cs.toString());
 	}
@@ -264,7 +283,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/hostnames") {
 	{
 		input = "0xxdeadbeef:100100100@localhost:34534,host-name:23443";
 		ClusterConnectionString cs(input);
-		ASSERT(cs.hasUnresolvedHostnames);
+		ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 		ASSERT(cs.hostnames.size() == 2);
 		ASSERT(input == cs.toString());
 	}
@@ -277,7 +296,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/hostnames") {
 		commented += "# asdfasdf ##";
 
 		ClusterConnectionString cs(commented);
-		ASSERT(cs.hasUnresolvedHostnames);
+		ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 		ASSERT(cs.hostnames.size() == 2);
 		ASSERT(input == cs.toString());
 	}
@@ -290,7 +309,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/hostnames") {
 		commented += "# asdfasdf ##";
 
 		ClusterConnectionString cs(commented);
-		ASSERT(cs.hasUnresolvedHostnames);
+		ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 		ASSERT(cs.hostnames.size() == 2);
 		ASSERT(input == cs.toString());
 	}
@@ -314,16 +333,16 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString") {
 	INetworkConnections::net()->addMockTCPEndpoint(hn2, port2, { address2 });
 
 	state ClusterConnectionString cs(hostnames, LiteralStringRef("TestCluster:0"));
-	ASSERT(cs.hasUnresolvedHostnames);
+	ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 	ASSERT(cs.hostnames.size() == 2);
 	ASSERT(cs.coordinators().size() == 0);
 	wait(cs.resolveHostnames());
-	ASSERT(!cs.hasUnresolvedHostnames);
+	ASSERT(cs.status == ClusterConnectionString::RESOLVED);
 	ASSERT(cs.hostnames.size() == 2);
 	ASSERT(cs.coordinators().size() == 2);
 	ASSERT(cs.toString() == connectionString);
 	cs.resetToUnresolved();
-	ASSERT(cs.hasUnresolvedHostnames);
+	ASSERT(cs.status == ClusterConnectionString::UNRESOLVED);
 	ASSERT(cs.hostnames.size() == 2);
 	ASSERT(cs.coordinators().size() == 0);
 	ASSERT(cs.toString() == connectionString);
@@ -422,7 +441,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
 }
 
 ClusterConnectionString::ClusterConnectionString(const std::vector<NetworkAddress>& servers, Key key)
-  : coords(servers) {
+  : status(RESOLVED), coords(servers) {
 	std::string keyString = key.toString();
 	parseKey(keyString);
 	connectionString = keyString + "@";
@@ -435,7 +454,7 @@ ClusterConnectionString::ClusterConnectionString(const std::vector<NetworkAddres
 }
 
 ClusterConnectionString::ClusterConnectionString(const std::vector<Hostname>& hosts, Key key)
-  : hasUnresolvedHostnames(true), hostnames(hosts) {
+  : status(UNRESOLVED), hostnames(hosts) {
 	std::string keyString = key.toString();
 	parseKey(keyString);
 	connectionString = keyString + "@";

From 59ac1dffb449138ee722bcc8ef3a4415e01a4cd9 Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Tue, 22 Feb 2022 09:36:30 -0800
Subject: [PATCH 093/210] change storage_migration_mode to
 storage_migration_type

---
 fdbcli/ConfigureCommand.actor.cpp | 2 +-
 fdbclient/ManagementAPI.actor.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp
index ab810454a4..3570f8dbd2 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.actor.cpp
@@ -176,7 +176,7 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 	case ConfigurationResult::STORAGE_MIGRATION_DISABLED:
 		fprintf(stderr,
 		        "ERROR: Storage engine type cannot be changed because "
-		        "storage_migration_mode=disabled.\n");
+		        "storage_migration_type=disabled.\n");
 		fprintf(stderr,
 		        "Type `configure perpetual_storage_wiggle=1 storage_migration_type=gradual' to enable gradual "
 		        "migration with the perpetual wiggle, or `configure "
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 4e26c9a16b..cf5763b8b5 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -169,7 +169,7 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			} else if (value == "gradual") {
 				type = StorageMigrationType::GRADUAL;
 			} else {
-				printf("Error: Only disabled|aggressive|gradual are valid for storage_migration_mode.\n");
+				printf("Error: Only disabled|aggressive|gradual are valid for storage_migration_type.\n");
 				return out;
 			}
 			out[p + key] = format("%d", type);

From 31d56b2f50cab40889c801b1defe70c2fde6b76d Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 22 Feb 2022 12:42:53 -0500
Subject: [PATCH 094/210] adjust knob setting, revert trace changes, and
 disable cache if onError retries transactions

---
 fdbclient/ClientKnobs.cpp     |  2 +-
 fdbclient/NativeAPI.actor.cpp | 10 ++++++----
 flow/Trace.cpp                | 20 ++++++++++----------
 flow/Trace.h                  |  2 +-
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 0b8190bef6..f962c9f9ba 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -118,7 +118,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( LOG_RANGE_BLOCK_SIZE, CORE_VERSIONSPERSECOND );
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 	init( MAX_VERSION_CACHE_LAG,                    0.1 );
-	init( MAX_PROXY_CONTACT_LAG,                    1.0 );
+	init( MAX_PROXY_CONTACT_LAG,                    0.2 );
 	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0. 
 	init( FORCE_GRV_CACHE_OFF,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4b52f15193..b26294786a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5776,7 +5776,9 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::USE_GRV_CACHE:
 		validateOptionValueNotPresent(value);
-		trState->options.useGrvCache = true;
+		if (trState->options.numErrors == 0) {
+			trState->options.useGrvCache = true;
+		}
 		break;
 
 	case FDBTransactionOptions::SKIP_GRV_CACHE:
@@ -6238,6 +6240,9 @@ uint32_t Transaction::getSize() {
 }
 
 Future<Void> Transaction::onError(Error const& e) {
+	if (g_network->isSimulated() && ++trState->numErrors % 10 == 0) {
+		TraceEvent(SevWarnAlways, "TransactionTooManyRetries").detail("NumRetries", trState->numErrors);
+	}
 	if (e.code() == error_code_success) {
 		return client_invalid_operation();
 	}
@@ -6272,9 +6277,6 @@ Future<Void> Transaction::onError(Error const& e) {
 		return delay(std::min(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, maxBackoff), trState->taskID);
 	}
 
-	if (g_network->isSimulated() && ++trState->numErrors % 10 == 0)
-		TraceEvent(SevWarnAlways, "TransactionTooManyRetries").detail("NumRetries", trState->numErrors);
-
 	return e;
 }
 ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRange keys);
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 4aafd83397..7aabc10a60 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -927,16 +927,16 @@ bool TraceEvent::init() {
 	enabled = enabled && (!g_network || severity >= FLOW_KNOBS->MIN_TRACE_SEVERITY);
 
 	// Backstop to throttle very spammy trace events
-	// if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
-	// 	if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
-	// 		enabled = false;
-	// 		TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
-	// 		    .suppressFor(5);
-	// 	} else {
-	// 		traceEventThrottlerCache->addAndExpire(
-	// 		    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
-	// 	}
-	// }
+	if (enabled && g_network && !g_network->isSimulated() && severity > SevDebug && isNetworkThread()) {
+		if (traceEventThrottlerCache->isAboveThreshold(StringRef((uint8_t*)type, strlen(type)))) {
+			enabled = false;
+			TraceEvent(SevWarnAlways, std::string(TRACE_EVENT_THROTTLE_STARTING_TYPE).append(type).c_str())
+			    .suppressFor(5);
+		} else {
+			traceEventThrottlerCache->addAndExpire(
+			    StringRef((uint8_t*)type, strlen(type)), 1, now() + FLOW_KNOBS->TRACE_EVENT_THROTTLER_SAMPLE_EXPIRY);
+		}
+	}
 
 	if (enabled) {
 		tmpEventMetric = std::make_unique<DynamicEventMetric>(MetricNameRef());
diff --git a/flow/Trace.h b/flow/Trace.h
index 8779a16a13..b30b5405de 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -33,7 +33,7 @@
 #include "flow/Error.h"
 #include "flow/ITrace.h"
 
-#define TRACE_DEFAULT_ROLL_SIZE (10 << 24)
+#define TRACE_DEFAULT_ROLL_SIZE (10 << 20)
 #define TRACE_DEFAULT_MAX_LOGS_SIZE (10 * TRACE_DEFAULT_ROLL_SIZE)
 
 inline int fastrand() {

From b60cc02aa2d7951ab194bdb54faec977dffe183a Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 22 Feb 2022 13:10:30 -0500
Subject: [PATCH 095/210] fix compile error

---
 fdbclient/NativeAPI.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b26294786a..5a4207a9ca 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5776,7 +5776,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::USE_GRV_CACHE:
 		validateOptionValueNotPresent(value);
-		if (trState->options.numErrors == 0) {
+		if (trState->numErrors == 0) {
 			trState->options.useGrvCache = true;
 		}
 		break;

From ae071a72113c728e76904b4126c9b60f98032bf0 Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 22 Feb 2022 17:16:11 -0500
Subject: [PATCH 096/210] clean up debug trace lines

---
 fdbclient/NativeAPI.actor.cpp      |  2 --
 fdbserver/GrvProxyServer.actor.cpp | 11 -----------
 2 files changed, 13 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 5a4207a9ca..b25cf28927 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5960,11 +5960,9 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	trState->cx->lastProxyRequestTime = trState->startTime;
 	trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
 	if (rep.rkBatchThrottled) {
-		TraceEvent("DebugGrvLastRkBatchThrottled").detail("ReplyTime", replyTime);
 		trState->cx->lastRkBatchThrottleTime = replyTime;
 	}
 	if (rep.rkDefaultThrottled) {
-		TraceEvent("DebugGrvLastRkDefaultThrottled").detail("ReplyTime", replyTime);
 		trState->cx->lastRkDefaultThrottleTime = replyTime;
 	}
 	trState->cx->GRVLatencies.addSample(latency);
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index eeb97ee516..64ae555a3a 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -656,15 +656,9 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			}
 		}
 
-		TraceEvent("DebugGrvProxyThrottleTimes")
-		    .detail("LastBatchThrottled", format("%.6f", stats->lastBatchQueueThrottled))
-		    .detail("BatchThrottleStart", format("%.6f", stats->batchThrottleStartTime))
-		    .detail("LastDefaultThrottled", format("%.6f", stats->lastDefaultQueueThrottled))
-		    .detail("DefaultThrottleStart", format("%.6f", stats->defaultThrottleStartTime));
 		if (stats->lastBatchQueueThrottled) {
 			// Check if this throttling has been sustained for a certain amount of time to avoid false positives
 			if (now() - stats->batchThrottleStartTime > CLIENT_KNOBS->GRV_SUSTAINED_THROTTLING_THRESHOLD) {
-				TraceEvent("DebugGrvProxyBatchThrottled").detail("StartTime", stats->batchThrottleStartTime);
 				reply.rkBatchThrottled = true;
 			}
 		}
@@ -674,7 +668,6 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 				// Consider the batch queue throttled if the default is throttled
 				// to deal with a potential lull in activity for that priority.
 				// Avoids mistakenly thinking batch is unthrottled while default is still throttled.
-				TraceEvent("DebugGrvProxyDefaultThrottled").detail("StartTime", stats->defaultThrottleStartTime);
 				reply.rkBatchThrottled = true;
 				reply.rkDefaultThrottled = true;
 			}
@@ -876,23 +869,19 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			requestsToStart++;
 		}
 		if (!batchQueue.empty()) {
-			TraceEvent("DebugGrvProxyBatchQueueNonEmpty").detail("Size", batchQueue.size());
 			if (!grvProxyData->stats.lastBatchQueueThrottled) {
 				grvProxyData->stats.lastBatchQueueThrottled = true;
 				grvProxyData->stats.batchThrottleStartTime = now();
 			}
 		} else {
-			TraceEvent("DebugGrvProxyBatchQueueEmpty");
 			grvProxyData->stats.lastBatchQueueThrottled = false;
 		}
 		if (!defaultQueue.empty()) {
-			TraceEvent("DebugGrvProxyDefaultQueueNonEmpty").detail("Size", defaultQueue.size());
 			if (!grvProxyData->stats.lastDefaultQueueThrottled) {
 				grvProxyData->stats.lastDefaultQueueThrottled = true;
 				grvProxyData->stats.defaultThrottleStartTime = now();
 			}
 		} else {
-			TraceEvent("DebugGrvProxyDefaultQueueEmpty");
 			grvProxyData->stats.lastDefaultQueueThrottled = false;
 		}
 

From 481587a8c62f2ad8c642423bd201941bf24edd38 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Fri, 21 Jan 2022 00:42:03 -0800
Subject: [PATCH 097/210] Turn on hostname logic.

---
 fdbcli/CoordinatorsCommand.actor.cpp          |   1 +
 fdbcli/fdbcli.actor.cpp                       |   4 +-
 fdbclient/CoordinationInterface.h             |   2 +-
 fdbclient/MonitorLeader.actor.cpp             | 212 ++++++++++++------
 fdbclient/MonitorLeader.h                     |   5 +-
 fdbclient/NativeAPI.actor.cpp                 |  14 +-
 fdbserver/CoordinatedState.actor.cpp          |   4 +
 fdbserver/Coordination.actor.cpp              |  64 +++++-
 fdbserver/QuietDatabase.actor.cpp             |   7 +-
 fdbserver/SimulatedCluster.actor.cpp          |   4 +-
 fdbserver/fdbserver.actor.cpp                 |   2 +-
 fdbserver/worker.actor.cpp                    |  33 ++-
 .../SpecialKeySpaceCorrectness.actor.cpp      |  26 ++-
 13 files changed, 274 insertions(+), 104 deletions(-)

diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.actor.cpp
index d8bfe11f6c..261090c5ed 100644
--- a/fdbcli/CoordinatorsCommand.actor.cpp
+++ b/fdbcli/CoordinatorsCommand.actor.cpp
@@ -100,6 +100,7 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 				state std::vector<StringRef>::iterator t;
 				for (t = tokens.begin() + 1; t != tokens.end(); ++t) {
 					try {
+						// TODO(renxuan): add hostname parsing here.
 						auto const& addr = NetworkAddress::parse(t->toString());
 						if (new_coordinators_addresses.count(addr)) {
 							fprintf(stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 6aa6c2538b..0ca539e343 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1157,7 +1157,6 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 
 	state bool writeMode = false;
 
-	state std::string clusterConnectString;
 	state std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
 
 	state FdbOptions globalOptions;
@@ -1171,6 +1170,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	    ClusterConnectionFile::lookupClusterFileName(opt.clusterFile);
 	try {
 		ccf = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
+		if (ccf->hasUnresolvedHostnames()) {
+			wait(ccf->resolveHostnames());
+		}
 	} catch (Error& e) {
 		fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
 		return 1;
diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index 50860db0da..9873817a88 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -99,11 +99,11 @@ public:
 	AsyncTrigger resolveFinish;
 	std::vector<NetworkAddress> coords;
 	std::vector<Hostname> hostnames;
+	std::unordered_map<NetworkAddress, Hostname> networkAddressToHostname;
 
 private:
 	void parseConnString();
 	void parseKey(const std::string& key);
-	std::unordered_map<NetworkAddress, Hostname> networkAddressToHostname;
 	Key key, keyDesc;
 	std::string connectionString;
 };
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 4f237e1039..e6dbeccd1d 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -516,6 +516,7 @@ std::string ClusterConnectionString::toString() const {
 }
 
 ClientCoordinators::ClientCoordinators(Reference<IClusterConnectionRecord> ccr) : ccr(ccr) {
+	ASSERT(!ccr->hasUnresolvedHostnames());
 	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
 		clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@@ -544,15 +545,44 @@ ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
 
 // Nominee is the worker among all workers that are considered as leader by one coordinator
 // This function contacts a coordinator coord to ask who is its nominee.
-ACTOR Future<Void> monitorNominee(Key key,
-                                  ClientLeaderRegInterface coord,
-                                  AsyncTrigger* nomineeChange,
-                                  Optional<LeaderInfo>* info) {
+// Note: for coordinators whose NetworkAddress is parsed out of a hostname, a connection failure will cause this actor
+// to throw `coordinators_changed()` error
+ACTOR Future<Void> monitorNominee(
+    Key key,
+    ClientLeaderRegInterface coord,
+    AsyncTrigger* nomineeChange,
+    Optional<LeaderInfo>* info,
+    Optional<Hostname> hostname = Optional<Hostname>(),
+    Reference<IClusterConnectionRecord> connRecord = Reference<IClusterConnectionRecord>()) {
 	loop {
-		state Optional<LeaderInfo> li =
-		    wait(retryBrokenPromise(coord.getLeader,
-		                            GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
-		                            TaskPriority::CoordinationReply));
+		if (connRecord.isValid() && connRecord->hasUnresolvedHostnames()) {
+			wait(connRecord->resolveHostnames());
+		}
+		state Optional<LeaderInfo> li;
+
+		if (coord.getLeader.getEndpoint().getPrimaryAddress().fromHostname) {
+			state ErrorOr<Optional<LeaderInfo>> rep =
+			    wait(coord.getLeader.tryGetReply(GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
+			                                     TaskPriority::CoordinationReply));
+			if (rep.isError() && rep.getError().code() == error_code_request_maybe_delivered) {
+				// Connecting to nominee failed, most likely due to connection failed.
+				TraceEvent("CoordnitorChangedMonitorNominee")
+				    .detail("Hostname", hostname.present() ? hostname.get().toString() : "UnknownHostname")
+				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString());
+				// 50 milliseconds delay to prevent tight resolving loop due to outdated DNS cache
+				wait(delay(0.05));
+				throw coordinators_changed();
+			} else if (rep.present()) {
+				li = rep.get();
+			}
+		} else {
+			Optional<LeaderInfo> tmp =
+			    wait(retryBrokenPromise(coord.getLeader,
+			                            GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
+			                            TaskPriority::CoordinationReply));
+			li = tmp;
+		}
+
 		wait(Future<Void>(Void())); // Make sure we weren't cancelled
 
 		TraceEvent("GetLeaderReply")
@@ -627,53 +657,81 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterConnectionRecord> connRecord,
                                                            Reference<AsyncVar<Value>> outSerializedLeaderInfo,
                                                            MonitorLeaderInfo info) {
-	state ClientCoordinators coordinators(info.intermediateConnRecord);
-	state AsyncTrigger nomineeChange;
-	state std::vector<Optional<LeaderInfo>> nominees;
-	state Future<Void> allActors;
-
-	nominees.resize(coordinators.clientLeaderServers.size());
-
-	std::vector<Future<Void>> actors;
-	// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
-	actors.reserve(coordinators.clientLeaderServers.size());
-	for (int i = 0; i < coordinators.clientLeaderServers.size(); i++)
-		actors.push_back(
-		    monitorNominee(coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i]));
-	allActors = waitForAll(actors);
-
 	loop {
-		Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
-		TraceEvent("MonitorLeaderChange")
-		    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
-		if (leader.present()) {
-			if (leader.get().first.forward) {
-				TraceEvent("MonitorLeaderForwarding")
-				    .detail("NewConnStr", leader.get().first.serializedInfo.toString())
-				    .detail("OldConnStr", info.intermediateConnRecord->getConnectionString().toString())
-				    .trackLatest("MonitorLeaderForwarding");
-				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
-				    ClusterConnectionString(leader.get().first.serializedInfo.toString()));
-				return info;
-			}
-			if (connRecord != info.intermediateConnRecord) {
-				if (!info.hasConnected) {
-					TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
-					    .detail("ClusterFile", connRecord->toString())
-					    .detail("StoredConnectionString", connRecord->getConnectionString().toString())
-					    .detail("CurrentConnectionString",
-					            info.intermediateConnRecord->getConnectionString().toString());
-				}
-				connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
-				info.intermediateConnRecord = connRecord;
-			}
-
-			info.hasConnected = true;
-			connRecord->notifyConnected();
-
-			outSerializedLeaderInfo->set(leader.get().first.serializedInfo);
+		if (connRecord->hasUnresolvedHostnames()) {
+			wait(connRecord->resolveHostnames());
+		}
+		if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
+			wait(info.intermediateConnRecord->resolveHostnames());
+		}
+		state ClientCoordinators coordinators(info.intermediateConnRecord);
+		state AsyncTrigger nomineeChange;
+		state std::vector<Optional<LeaderInfo>> nominees;
+		state Future<Void> allActors;
+
+		nominees.resize(coordinators.clientLeaderServers.size());
+
+		state std::vector<Future<Void>> actors;
+		// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
+		actors.reserve(coordinators.clientLeaderServers.size());
+		for (int i = 0; i < coordinators.clientLeaderServers.size(); i++) {
+			Optional<Hostname> hostname;
+			auto r = connRecord->getConnectionString().networkAddressToHostname.find(
+			    coordinators.clientLeaderServers[i].getLeader.getEndpoint().getPrimaryAddress());
+			if (r != connRecord->getConnectionString().networkAddressToHostname.end()) {
+				hostname = r->second;
+			}
+			actors.push_back(monitorNominee(coordinators.clusterKey,
+			                                coordinators.clientLeaderServers[i],
+			                                &nomineeChange,
+			                                &nominees[i],
+			                                hostname,
+			                                connRecord));
+		}
+		allActors = waitForAll(actors);
+
+		loop {
+			Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
+			TraceEvent("MonitorLeaderChange")
+			    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
+			if (leader.present()) {
+				if (leader.get().first.forward) {
+					TraceEvent("MonitorLeaderForwarding")
+					    .detail("NewConnStr", leader.get().first.serializedInfo.toString())
+					    .detail("OldConnStr", info.intermediateConnRecord->getConnectionString().toString())
+					    .trackLatest("MonitorLeaderForwarding");
+					info.intermediateConnRecord = connRecord->makeIntermediateRecord(
+					    ClusterConnectionString(leader.get().first.serializedInfo.toString()));
+					return info;
+				}
+				if (connRecord != info.intermediateConnRecord) {
+					if (!info.hasConnected) {
+						TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
+						    .detail("ClusterFile", connRecord->toString())
+						    .detail("StoredConnectionString", connRecord->getConnectionString().toString())
+						    .detail("CurrentConnectionString",
+						            info.intermediateConnRecord->getConnectionString().toString());
+					}
+					connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
+					info.intermediateConnRecord = connRecord;
+				}
+
+				info.hasConnected = true;
+				connRecord->notifyConnected();
+
+				outSerializedLeaderInfo->set(leader.get().first.serializedInfo);
+			}
+			try {
+				wait(nomineeChange.onTrigger() || allActors);
+			} catch (Error& e) {
+				if (e.code() == error_code_coordinators_changed) {
+					connRecord->getMutableConnectionString()->resetToUnresolved();
+					break;
+				} else {
+					throw e;
+				}
+			}
 		}
-		wait(nomineeChange.onTrigger() || allActors);
 	}
 }
 
@@ -793,8 +851,8 @@ ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterCo
 			when(ClientDBInfo ni =
 			         wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
 				TraceEvent("GetClientInfoFromLeaderGotClientInfo", knownLeader->get().get().clientInterface.id())
-				    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID())
-				    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID())
+				    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].address().toString() : "")
+				    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].address().toString() : "")
 				    .detail("ClientID", ni.id);
 				clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
 			}
@@ -806,7 +864,8 @@ ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterCo
 ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
                                                  std::vector<NetworkAddress> coordinators,
                                                  ClientData* clientData,
-                                                 Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
+                                                 Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo,
+                                                 Reference<AsyncVar<Void>> coordinatorsChanged) {
 	state std::vector<ClientLeaderRegInterface> clientLeaderServers;
 	state AsyncTrigger nomineeChange;
 	state std::vector<Optional<LeaderInfo>> nominees;
@@ -854,7 +913,14 @@ ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
 				leaderInfo->set(leader.get().first);
 			}
 		}
-		wait(nomineeChange.onTrigger() || allActors);
+		try {
+			wait(nomineeChange.onTrigger() || allActors);
+		} catch (Error& e) {
+			if (e.code() == error_code_coordinators_changed) {
+				coordinatorsChanged->trigger();
+			}
+			return Void();
+		}
 	}
 }
 
@@ -983,9 +1049,13 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			successIndex = index;
 		} else {
 			TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller
+			if (rep.getError().code() == error_code_coordinators_changed) {
+				throw coordinators_changed();
+			}
 			index = (index + 1) % addrs.size();
 			if (index == successIndex) {
 				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
+				throw coordinators_changed();
 			}
 		}
 	}
@@ -997,16 +1067,30 @@ ACTOR Future<Void> monitorProxies(
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup) {
+	if (connRecord->get()->hasUnresolvedHostnames()) {
+		wait(connRecord->get()->resolveHostnames());
+	}
 	state MonitorLeaderInfo info(connRecord->get());
 	loop {
-		choose {
-			when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
-			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
-				info = _info;
+		try {
+			if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
+				wait(info.intermediateConnRecord->resolveHostnames());
 			}
-			when(wait(connRecord->onChange())) {
-				info.hasConnected = false;
-				info.intermediateConnRecord = connRecord->get();
+			choose {
+				when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
+				         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
+					info = _info;
+				}
+				when(wait(connRecord->onChange())) {
+					info.hasConnected = false;
+					info.intermediateConnRecord = connRecord->get();
+				}
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_coordinators_changed) {
+				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+			} else {
+				throw e;
 			}
 		}
 	}
diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h
index 2903ff51b9..4aa3e5ff7f 100644
--- a/fdbclient/MonitorLeader.h
+++ b/fdbclient/MonitorLeader.h
@@ -74,10 +74,11 @@ Future<Void> monitorLeader(Reference<IClusterConnectionRecord> const& connFile,
 // This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect
 // nominees, the nominee with the most nomination is the leader, and collects client data from the leader. This function
 // also monitors the change of the leader.
-Future<Void> monitorLeaderAndGetClientInfo(Value const& key,
+Future<Void> monitorLeaderAndGetClientInfo(Key const& clusterKey,
                                            std::vector<NetworkAddress> const& coordinators,
                                            ClientData* const& clientData,
-                                           Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
+                                           Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo,
+                                           Reference<AsyncVar<Void>> const& coordinatorsChanged);
 
 Future<Void> monitorProxies(
     Reference<AsyncVar<Reference<IClusterConnectionRecord>>> const& connRecord,
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b2019b989b..0e57ade435 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -732,16 +732,18 @@ Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
 
 ACTOR static Future<Void> monitorClientDBInfoChange(DatabaseContext* cx,
                                                     Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
-                                                    AsyncTrigger* proxyChangeTrigger) {
+                                                    AsyncTrigger* proxiesChangeTrigger) {
 	state std::vector<CommitProxyInterface> curCommitProxies;
 	state std::vector<GrvProxyInterface> curGrvProxies;
 	state ActorCollection actors(false);
+	state Future<Void> clientDBInfoOnChange = clientDBInfo->onChange();
 	curCommitProxies = clientDBInfo->get().commitProxies;
 	curGrvProxies = clientDBInfo->get().grvProxies;
 
 	loop {
 		choose {
-			when(wait(clientDBInfo->onChange())) {
+			when(wait(clientDBInfoOnChange)) {
+				clientDBInfoOnChange = clientDBInfo->onChange();
 				if (clientDBInfo->get().commitProxies != curCommitProxies ||
 				    clientDBInfo->get().grvProxies != curGrvProxies) {
 					// This condition is a bit complicated. Here we want to verify that we're unable to receive a read
@@ -758,7 +760,7 @@ ACTOR static Future<Void> monitorClientDBInfoChange(DatabaseContext* cx,
 					}
 					curCommitProxies = clientDBInfo->get().commitProxies;
 					curGrvProxies = clientDBInfo->get().grvProxies;
-					proxyChangeTrigger->trigger();
+					proxiesChangeTrigger->trigger();
 				}
 			}
 			when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); }
@@ -5713,9 +5715,10 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanID parentSpan,
 	loop {
 		try {
 			state GetReadVersionRequest req(span.context, transactionCount, priority, flags, tags, debugID);
+			state Future<Void> onProxiesChanged = cx->onProxiesChanged();
 
 			choose {
-				when(wait(cx->onProxiesChanged())) {}
+				when(wait(onProxiesChanged)) { onProxiesChanged = cx->onProxiesChanged(); }
 				when(GetReadVersionReply v =
 				         wait(basicLoadBalance(cx->getGrvProxies(UseProvisionalProxies(
 				                                   flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES)),
@@ -6881,6 +6884,9 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 		throw;
 	}
 	TraceEvent("ExclusionSafetyCheckCoordinators").log();
+	if (cx->getConnectionRecord()->hasUnresolvedHostnames()) {
+		wait(cx->getConnectionRecord()->resolveHostnames());
+	}
 	state ClientCoordinators coordinatorList(cx->getConnectionRecord());
 	state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	leaderServers.reserve(coordinatorList.clientLeaderServers.size());
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/CoordinatedState.actor.cpp
index 4ffdd057a2..75add352c7 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/CoordinatedState.actor.cpp
@@ -359,5 +359,9 @@ Future<Void> MovableCoordinatedState::setExclusive(Value v) {
 	return impl->setExclusive(v);
 }
 Future<Void> MovableCoordinatedState::move(ClusterConnectionString const& nc) {
+	// We assume the new connection string contains only IP addresses. Even `coordinators` command in fdbcli is going to
+	// support hostnames, those hostnames should have been resolved in place and will be treated as IP address
+	// afterwards, i.e. no automatic re-resolve as that of the hostname used in cluster file.
+	ASSERT(nc.hostnames.size() == 0);
 	return MovableCoordinatedStateImpl::move(impl.get(), nc);
 }
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index dc3d1fa1f2..5187edf933 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -96,6 +96,7 @@ LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : Client
 }
 
 ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
+	ASSERT(!ccr->hasUnresolvedHostnames());
 	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s) {
 		leaderElectionServers.emplace_back(*s);
@@ -205,8 +206,11 @@ ACTOR Future<Void> openDatabase(ClientData* db,
                                 int* clientCount,
                                 Reference<AsyncVar<bool>> hasConnectedClients,
                                 OpenDatabaseCoordRequest req,
-                                Future<Void> checkStuck) {
+                                Future<Void> checkStuck,
+                                Reference<AsyncVar<Void>> coordinatorsChanged) {
 	state ErrorOr<CachedSerialization<ClientDBInfo>> replyContents;
+	state Future<Void> coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+	state Future<Void> clientInfoOnChange = db->clientInfo->onChange();
 
 	++(*clientCount);
 	hasConnectedClients->set(true);
@@ -223,7 +227,15 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 				replyContents = failed_to_progress();
 				break;
 			}
-			when(wait(yieldedFuture(db->clientInfo->onChange()))) { replyContents = db->clientInfo->get(); }
+			when(wait(yieldedFuture(clientInfoOnChange))) {
+				clientInfoOnChange = db->clientInfo->onChange();
+				replyContents = db->clientInfo->get();
+			}
+			when(wait(coordinatorsChangedOnChange)) {
+				coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+				replyContents = coordinators_changed();
+				break;
+			}
 			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) {
 				if (db->clientInfo->get().read().id.isValid()) {
 					replyContents = db->clientInfo->get();
@@ -254,18 +266,33 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
                                        Reference<AsyncVar<bool>> hasConnectedClients,
                                        Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader,
-                                       ElectionResultRequest req) {
+                                       ElectionResultRequest req,
+                                       Reference<AsyncVar<Void>> coordinatorsChanged) {
+	state bool _coordinatorsChanged = false;
+	state Future<Void> coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+	state Future<Void> currentElectedLeaderOnChange = currentElectedLeader->onChange();
 	++(*clientCount);
 	hasConnectedClients->set(true);
 
 	while (!currentElectedLeader->get().present() || req.knownLeader == currentElectedLeader->get().get().changeID) {
 		choose {
-			when(wait(yieldedFuture(currentElectedLeader->onChange()))) {}
+			when(wait(yieldedFuture(currentElectedLeaderOnChange))) {
+				currentElectedLeaderOnChange = currentElectedLeader->onChange();
+			}
+			when(wait(coordinatorsChangedOnChange)) {
+				coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+				_coordinatorsChanged = true;
+				break;
+			}
 			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) { break; }
 		}
 	}
 
-	req.reply.send(currentElectedLeader->get());
+	if (_coordinatorsChanged) {
+		req.reply.sendError(coordinators_changed());
+	} else {
+		req.reply.send(currentElectedLeader->get());
+	}
 
 	if (--(*clientCount) == 0) {
 		hasConnectedClients->set(false);
@@ -296,6 +323,9 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 	state Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader =
 	    makeReference<AsyncVar<Optional<LeaderInfo>>>();
 	state LivenessChecker canConnectToLeader(SERVER_KNOBS->COORDINATOR_LEADER_CONNECTION_TIMEOUT);
+	state Reference<AsyncVar<Void>> coordinatorsChanged = makeReference<AsyncVar<Void>>();
+	state Future<Void> coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+	state Future<Void> hasConnectedClientsOnChange = hasConnectedClients->onChange();
 
 	loop choose {
 		when(OpenDatabaseCoordRequest req = waitNext(interf.openDatabase.getFuture())) {
@@ -306,10 +336,14 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 			} else {
 				if (!leaderMon.isValid()) {
 					leaderMon = monitorLeaderAndGetClientInfo(
-					    req.clusterKey, req.coordinators, &clientData, currentElectedLeader);
+					    req.clusterKey, req.coordinators, &clientData, currentElectedLeader, coordinatorsChanged);
 				}
-				actors.add(
-				    openDatabase(&clientData, &clientCount, hasConnectedClients, req, canConnectToLeader.checkStuck()));
+				actors.add(openDatabase(&clientData,
+				                        &clientCount,
+				                        hasConnectedClients,
+				                        req,
+				                        canConnectToLeader.checkStuck(),
+				                        coordinatorsChanged));
 			}
 		}
 		when(ElectionResultRequest req = waitNext(interf.electionResult.getFuture())) {
@@ -318,10 +352,11 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				req.reply.send(currentElectedLeader->get());
 			} else {
 				if (!leaderMon.isValid()) {
-					leaderMon =
-					    monitorLeaderAndGetClientInfo(req.key, req.coordinators, &clientData, currentElectedLeader);
+					leaderMon = monitorLeaderAndGetClientInfo(
+					    req.key, req.coordinators, &clientData, currentElectedLeader, coordinatorsChanged);
 				}
-				actors.add(remoteMonitorLeader(&clientCount, hasConnectedClients, currentElectedLeader, req));
+				actors.add(remoteMonitorLeader(
+				    &clientCount, hasConnectedClients, currentElectedLeader, req, coordinatorsChanged));
 			}
 		}
 		when(GetLeaderRequest req = waitNext(interf.getLeader.getFuture())) {
@@ -454,13 +489,18 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				notify.pop_front();
 			}
 		}
-		when(wait(hasConnectedClients->onChange())) {
+		when(wait(hasConnectedClientsOnChange)) {
+			hasConnectedClientsOnChange = hasConnectedClients->onChange();
 			if (!hasConnectedClients->get() && !nextInterval.isValid()) {
 				TraceEvent("LeaderRegisterUnneeded").detail("Key", key);
 				return Void();
 			}
 		}
 		when(wait(actors.getResult())) {}
+		when(wait(coordinatorsChangedOnChange)) {
+			leaderMon = Future<Void>();
+			coordinatorsChangedOnChange = coordinatorsChanged->onChange();
+		}
 	}
 }
 
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index b3f6fc1c7a..336fa098f5 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -158,8 +158,11 @@ ACTOR Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx,
 	if (!coordinators.present()) {
 		throw operation_failed();
 	}
-	std::vector<NetworkAddress> coordinatorsAddr =
-	    ClusterConnectionString(coordinators.get().toString()).coordinators();
+	state ClusterConnectionString ccs(coordinators.get().toString());
+	if (ccs.hasUnresolvedHostnames) {
+		wait(ccs.resolveHostnames());
+	}
+	std::vector<NetworkAddress> coordinatorsAddr = ccs.coordinators();
 	std::set<NetworkAddress> coordinatorsAddrSet;
 	for (const auto& addr : coordinatorsAddr) {
 		TraceEvent(SevDebug, "CoordinatorAddress").detail("Addr", addr);
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index a49207457c..5fa01027ae 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -1905,8 +1905,8 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	TEST(useIPv6); // Use IPv6
 	TEST(!useIPv6); // Use IPv4
 
-	// TODO(renxuan): Use hostname 25% of the time, unless it is disabled
-	bool useHostname = false; // !testConfig.disableHostname && deterministicRandom()->random01() < 0.25;
+	// Use hostname 25% of the time, unless it is disabled
+	bool useHostname = !testConfig.disableHostname && deterministicRandom()->random01() < 0.25;
 	TEST(useHostname); // Use hostname
 	TEST(!useHostname); // Use IP address
 	NetworkAddressFromHostname fromHostname =
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 8bcd61c943..4f1f32eb5e 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -834,7 +834,7 @@ std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
 	NetworkAddressList listenNetworkAddresses;
 
 	auto& coordinators = connectionRecord.getConnectionString().coordinators();
-	ASSERT(coordinators.size() > 0);
+	ASSERT(coordinators.size() + connectionRecord.getConnectionString().hostnames.size() > 0);
 
 	for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
 		const std::string& publicAddressStr = publicAddressStrs[ii];
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 0895c1ecbb..85d623928e 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -2308,10 +2308,11 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 ACTOR Future<Void> extractClusterInterface(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> in,
                                            Reference<AsyncVar<Optional<ClusterInterface>>> out) {
 	loop {
-		if (in->get().present())
+		if (in->get().present()) {
 			out->set(in->get().get().clientInterface);
-		else
+		} else {
 			out->set(Optional<ClusterInterface>());
+		}
 		wait(in->onChange());
 	}
 }
@@ -2509,9 +2510,14 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 			}
 			successIndex = index;
 		} else {
+			if (leader.isError() && leader.getError().code() == error_code_coordinators_changed) {
+				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+				throw coordinators_changed();
+			}
 			index = (index + 1) % addrs.size();
 			if (index == successIndex) {
 				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
+				throw coordinators_changed();
 			}
 		}
 	}
@@ -2519,11 +2525,25 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 
 ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClusterConnectionRecord> connRecord,
                                                                  Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
+	if (connRecord->hasUnresolvedHostnames()) {
+		wait(connRecord->resolveHostnames());
+	}
 	state MonitorLeaderInfo info(connRecord);
 	loop {
-		MonitorLeaderInfo _info =
-		    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info));
-		info = _info;
+		try {
+			if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
+				wait(info.intermediateConnRecord->resolveHostnames());
+			}
+			MonitorLeaderInfo _info =
+			    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info));
+			info = _info;
+		} catch (Error& e) {
+			if (e.code() == error_code_coordinators_changed) {
+				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+			} else {
+				throw e;
+			}
+		}
 	}
 }
 
@@ -2657,6 +2677,9 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 	actors.push_back(serveProcess());
 
 	try {
+		if (connRecord->hasUnresolvedHostnames()) {
+			wait(connRecord->resolveHostnames());
+		}
 		ServerCoordinators coordinators(connRecord);
 		if (g_network->isSimulated()) {
 			whitelistBinPaths = ",, random_path,  /bin/snap_create.sh,,";
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index de8af3c4f1..4534f97965 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -926,10 +926,13 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    wait(tx->get(LiteralStringRef("processes")
 				                     .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
 				ASSERT(coordinator_processes_key.present());
-				std::vector<std::string> process_addresses;
+				state std::vector<std::string> process_addresses;
 				boost::split(
 				    process_addresses, coordinator_processes_key.get().toString(), [](char c) { return c == ','; });
-				ASSERT(process_addresses.size() == cs.coordinators().size());
+				ASSERT(process_addresses.size() == cs.coordinators().size() + cs.hostnames.size());
+				if (cs.hasUnresolvedHostnames) {
+					wait(cs.resolveHostnames());
+				}
 				// compare the coordinator process network addresses one by one
 				for (const auto& network_address : cs.coordinators()) {
 					ASSERT(std::find(process_addresses.begin(), process_addresses.end(), network_address.toString()) !=
@@ -970,16 +973,15 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					    old_coordinators_processes, processes_key.get().toString(), [](char c) { return c == ','; });
 					// pick up one non-coordinator process if possible
 					std::vector<ProcessData> workers = wait(getWorkers(&tx->getTransaction()));
+					std::string old_coordinators_processes_string = describe(old_coordinators_processes);
 					TraceEvent(SevDebug, "CoordinatorsManualChange")
-					    .detail("OldCoordinators", describe(old_coordinators_processes))
+					    .detail("OldCoordinators", old_coordinators_processes_string)
 					    .detail("WorkerSize", workers.size());
 					if (workers.size() > old_coordinators_processes.size()) {
 						loop {
 							auto worker = deterministicRandom()->randomChoice(workers);
 							new_coordinator_process = worker.address.toString();
-							if (std::find(old_coordinators_processes.begin(),
-							              old_coordinators_processes.end(),
-							              worker.address.toString()) == old_coordinators_processes.end()) {
+							if (old_coordinators_processes_string.find(new_coordinator_process) == std::string::npos) {
 								break;
 							}
 						}
@@ -1049,10 +1051,14 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					Optional<Value> res = wait(tx->get(coordinatorsKey));
 					ASSERT(res.present()); // Otherwise, database is in a bad state
-					ClusterConnectionString cs(res.get().toString());
-					ASSERT(cs.coordinators().size() == old_coordinators_processes.size() + 1);
+					state ClusterConnectionString csNew(res.get().toString());
+					ASSERT(csNew.coordinators().size() + csNew.hostnames.size() ==
+					       old_coordinators_processes.size() + 1);
+					if (csNew.hasUnresolvedHostnames) {
+						wait(csNew.resolveHostnames());
+					}
 					// verify the coordinators' addresses
-					for (const auto& network_address : cs.coordinators()) {
+					for (const auto& network_address : csNew.coordinators()) {
 						std::string address_str = network_address.toString();
 						ASSERT(std::find(old_coordinators_processes.begin(),
 						                 old_coordinators_processes.end(),
@@ -1060,7 +1066,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 						       new_coordinator_process == address_str);
 					}
 					// verify the cluster decription
-					ASSERT(new_cluster_description == cs.clusterKeyName().toString());
+					ASSERT(new_cluster_description == csNew.clusterKeyName().toString());
 					tx->reset();
 				} catch (Error& e) {
 					wait(tx->onError(e));

From 8eb7a1040478206fa0e35263e0d50531229edfc4 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Tue, 25 Jan 2022 18:53:23 -0800
Subject: [PATCH 098/210] Address comments.

---
 fdbclient/MonitorLeader.actor.cpp             | 46 +++++++++----------
 fdbserver/CoordinatedState.actor.cpp          |  4 --
 fdbserver/Coordination.actor.cpp              |  6 +--
 fdbserver/worker.actor.cpp                    |  4 +-
 .../SpecialKeySpaceCorrectness.actor.cpp      |  3 +-
 5 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index e6dbeccd1d..65f69e7d24 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -547,31 +547,31 @@ ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
 // This function contacts a coordinator coord to ask who is its nominee.
 // Note: for coordinators whose NetworkAddress is parsed out of a hostname, a connection failure will cause this actor
 // to throw `coordinators_changed()` error
-ACTOR Future<Void> monitorNominee(
-    Key key,
-    ClientLeaderRegInterface coord,
-    AsyncTrigger* nomineeChange,
-    Optional<LeaderInfo>* info,
-    Optional<Hostname> hostname = Optional<Hostname>(),
-    Reference<IClusterConnectionRecord> connRecord = Reference<IClusterConnectionRecord>()) {
+ACTOR Future<Void> monitorNominee(Key key,
+                                  ClientLeaderRegInterface coord,
+                                  AsyncTrigger* nomineeChange,
+                                  Optional<LeaderInfo>* info,
+                                  Optional<Hostname> hostname = Optional<Hostname>()) {
 	loop {
-		if (connRecord.isValid() && connRecord->hasUnresolvedHostnames()) {
-			wait(connRecord->resolveHostnames());
-		}
 		state Optional<LeaderInfo> li;
 
 		if (coord.getLeader.getEndpoint().getPrimaryAddress().fromHostname) {
 			state ErrorOr<Optional<LeaderInfo>> rep =
 			    wait(coord.getLeader.tryGetReply(GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
 			                                     TaskPriority::CoordinationReply));
-			if (rep.isError() && rep.getError().code() == error_code_request_maybe_delivered) {
+			if (rep.isError()) {
 				// Connecting to nominee failed, most likely due to connection failed.
-				TraceEvent("CoordnitorChangedMonitorNominee")
+				TraceEvent("MonitorNomineeError")
 				    .detail("Hostname", hostname.present() ? hostname.get().toString() : "UnknownHostname")
-				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString());
-				// 50 milliseconds delay to prevent tight resolving loop due to outdated DNS cache
-				wait(delay(0.05));
-				throw coordinators_changed();
+				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString())
+				    .error(rep.getError());
+				if (rep.getError().code() == error_code_request_maybe_delivered) {
+					// 50 milliseconds delay to prevent tight resolving loop due to outdated DNS cache
+					wait(delay(0.05));
+					throw coordinators_changed();
+				} else {
+					throw rep.getError();
+				}
 			} else if (rep.present()) {
 				li = rep.get();
 			}
@@ -681,12 +681,8 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 			if (r != connRecord->getConnectionString().networkAddressToHostname.end()) {
 				hostname = r->second;
 			}
-			actors.push_back(monitorNominee(coordinators.clusterKey,
-			                                coordinators.clientLeaderServers[i],
-			                                &nomineeChange,
-			                                &nominees[i],
-			                                hostname,
-			                                connRecord));
+			actors.push_back(monitorNominee(
+			    coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i], hostname));
 		}
 		allActors = waitForAll(actors);
 
@@ -725,7 +721,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 				wait(nomineeChange.onTrigger() || allActors);
 			} catch (Error& e) {
 				if (e.code() == error_code_coordinators_changed) {
-					connRecord->getMutableConnectionString()->resetToUnresolved();
+					connRecord->getConnectionString().resetToUnresolved();
 					break;
 				} else {
 					throw e;
@@ -919,7 +915,7 @@ ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
 			if (e.code() == error_code_coordinators_changed) {
 				coordinatorsChanged->trigger();
 			}
-			return Void();
+			throw e;
 		}
 	}
 }
@@ -1088,7 +1084,7 @@ ACTOR Future<Void> monitorProxies(
 			}
 		} catch (Error& e) {
 			if (e.code() == error_code_coordinators_changed) {
-				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+				info.intermediateConnRecord->getConnectionString().resetToUnresolved();
 			} else {
 				throw e;
 			}
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/CoordinatedState.actor.cpp
index 75add352c7..4ffdd057a2 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/CoordinatedState.actor.cpp
@@ -359,9 +359,5 @@ Future<Void> MovableCoordinatedState::setExclusive(Value v) {
 	return impl->setExclusive(v);
 }
 Future<Void> MovableCoordinatedState::move(ClusterConnectionString const& nc) {
-	// We assume the new connection string contains only IP addresses. Even `coordinators` command in fdbcli is going to
-	// support hostnames, those hostnames should have been resolved in place and will be treated as IP address
-	// afterwards, i.e. no automatic re-resolve as that of the hostname used in cluster file.
-	ASSERT(nc.hostnames.size() == 0);
 	return MovableCoordinatedStateImpl::move(impl.get(), nc);
 }
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index 5187edf933..46ae19bd0a 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -268,7 +268,7 @@ ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
                                        Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader,
                                        ElectionResultRequest req,
                                        Reference<AsyncVar<Void>> coordinatorsChanged) {
-	state bool _coordinatorsChanged = false;
+	state bool coordinatorsChangeDetected = false;
 	state Future<Void> coordinatorsChangedOnChange = coordinatorsChanged->onChange();
 	state Future<Void> currentElectedLeaderOnChange = currentElectedLeader->onChange();
 	++(*clientCount);
@@ -281,14 +281,14 @@ ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
 			}
 			when(wait(coordinatorsChangedOnChange)) {
 				coordinatorsChangedOnChange = coordinatorsChanged->onChange();
-				_coordinatorsChanged = true;
+				coordinatorsChangeDetected = true;
 				break;
 			}
 			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) { break; }
 		}
 	}
 
-	if (_coordinatorsChanged) {
+	if (coordinatorsChangeDetected) {
 		req.reply.sendError(coordinators_changed());
 	} else {
 		req.reply.send(currentElectedLeader->get());
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 85d623928e..b1ed805447 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -2511,7 +2511,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 			successIndex = index;
 		} else {
 			if (leader.isError() && leader.getError().code() == error_code_coordinators_changed) {
-				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+				info.intermediateConnRecord->getConnectionString().resetToUnresolved();
 				throw coordinators_changed();
 			}
 			index = (index + 1) % addrs.size();
@@ -2539,7 +2539,7 @@ ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClus
 			info = _info;
 		} catch (Error& e) {
 			if (e.code() == error_code_coordinators_changed) {
-				info.intermediateConnRecord->getMutableConnectionString()->resetToUnresolved();
+				info.intermediateConnRecord->getConnectionString().resetToUnresolved();
 			} else {
 				throw e;
 			}
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index 4534f97965..9ca399fa1c 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -1052,11 +1052,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					Optional<Value> res = wait(tx->get(coordinatorsKey));
 					ASSERT(res.present()); // Otherwise, database is in a bad state
 					state ClusterConnectionString csNew(res.get().toString());
-					ASSERT(csNew.coordinators().size() + csNew.hostnames.size() ==
-					       old_coordinators_processes.size() + 1);
 					if (csNew.hasUnresolvedHostnames) {
 						wait(csNew.resolveHostnames());
 					}
+					ASSERT(csNew.coordinators().size() == old_coordinators_processes.size() + 1);
 					// verify the coordinators' addresses
 					for (const auto& network_address : csNew.coordinators()) {
 						std::string address_str = network_address.toString();

From 622d89b55224a0030569e93ffb93c725b95feb4e Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Mon, 21 Feb 2022 11:53:54 -0800
Subject: [PATCH 099/210] Rebase on main.

Since we changed ClusterConnectionString's status flag from boolean to enum in #6422, we need to update this PR correspondingly.
---
 fdbcli/fdbcli.actor.cpp                        |  4 +---
 fdbclient/MonitorLeader.actor.cpp              | 18 +++++-------------
 fdbclient/NativeAPI.actor.cpp                  |  4 +---
 fdbserver/Coordination.actor.cpp               |  2 +-
 fdbserver/QuietDatabase.actor.cpp              |  4 +---
 fdbserver/worker.actor.cpp                     | 12 +++---------
 .../SpecialKeySpaceCorrectness.actor.cpp       |  8 ++------
 7 files changed, 14 insertions(+), 38 deletions(-)

diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 0ca539e343..c94f912845 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1170,9 +1170,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	    ClusterConnectionFile::lookupClusterFileName(opt.clusterFile);
 	try {
 		ccf = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
-		if (ccf->hasUnresolvedHostnames()) {
-			wait(ccf->resolveHostnames());
-		}
+		wait(ccf->resolveHostnames());
 	} catch (Error& e) {
 		fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
 		return 1;
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 65f69e7d24..ded347878e 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -516,7 +516,7 @@ std::string ClusterConnectionString::toString() const {
 }
 
 ClientCoordinators::ClientCoordinators(Reference<IClusterConnectionRecord> ccr) : ccr(ccr) {
-	ASSERT(!ccr->hasUnresolvedHostnames());
+	ASSERT(ccr->connectionStringStatus() == ClusterConnectionString::RESOLVED);
 	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
 		clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@@ -658,12 +658,8 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
                                                            Reference<AsyncVar<Value>> outSerializedLeaderInfo,
                                                            MonitorLeaderInfo info) {
 	loop {
-		if (connRecord->hasUnresolvedHostnames()) {
-			wait(connRecord->resolveHostnames());
-		}
-		if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
-			wait(info.intermediateConnRecord->resolveHostnames());
-		}
+		wait(connRecord->resolveHostnames());
+		wait(info.intermediateConnRecord->resolveHostnames());
 		state ClientCoordinators coordinators(info.intermediateConnRecord);
 		state AsyncTrigger nomineeChange;
 		state std::vector<Optional<LeaderInfo>> nominees;
@@ -1063,15 +1059,11 @@ ACTOR Future<Void> monitorProxies(
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup) {
-	if (connRecord->get()->hasUnresolvedHostnames()) {
-		wait(connRecord->get()->resolveHostnames());
-	}
+	wait(connRecord->get()->resolveHostnames());
 	state MonitorLeaderInfo info(connRecord->get());
 	loop {
 		try {
-			if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
-				wait(info.intermediateConnRecord->resolveHostnames());
-			}
+			wait(info.intermediateConnRecord->resolveHostnames());
 			choose {
 				when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
 				         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 0e57ade435..ace47c6c97 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6884,9 +6884,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 		throw;
 	}
 	TraceEvent("ExclusionSafetyCheckCoordinators").log();
-	if (cx->getConnectionRecord()->hasUnresolvedHostnames()) {
-		wait(cx->getConnectionRecord()->resolveHostnames());
-	}
+	wait(cx->getConnectionRecord()->resolveHostnames());
 	state ClientCoordinators coordinatorList(cx->getConnectionRecord());
 	state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	leaderServers.reserve(coordinatorList.clientLeaderServers.size());
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index 46ae19bd0a..d334de8bc5 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -96,7 +96,7 @@ LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : Client
 }
 
 ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
-	ASSERT(!ccr->hasUnresolvedHostnames());
+	ASSERT(ccr->connectionStringStatus() == ClusterConnectionString::RESOLVED);
 	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s) {
 		leaderElectionServers.emplace_back(*s);
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index 336fa098f5..56384d15bd 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -159,9 +159,7 @@ ACTOR Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx,
 		throw operation_failed();
 	}
 	state ClusterConnectionString ccs(coordinators.get().toString());
-	if (ccs.hasUnresolvedHostnames) {
-		wait(ccs.resolveHostnames());
-	}
+	wait(ccs.resolveHostnames());
 	std::vector<NetworkAddress> coordinatorsAddr = ccs.coordinators();
 	std::set<NetworkAddress> coordinatorsAddrSet;
 	for (const auto& addr : coordinatorsAddr) {
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index b1ed805447..70f0e90404 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -2525,15 +2525,11 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 
 ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClusterConnectionRecord> connRecord,
                                                                  Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
-	if (connRecord->hasUnresolvedHostnames()) {
-		wait(connRecord->resolveHostnames());
-	}
+	wait(connRecord->resolveHostnames());
 	state MonitorLeaderInfo info(connRecord);
 	loop {
 		try {
-			if (info.intermediateConnRecord->hasUnresolvedHostnames()) {
-				wait(info.intermediateConnRecord->resolveHostnames());
-			}
+			wait(info.intermediateConnRecord->resolveHostnames());
 			MonitorLeaderInfo _info =
 			    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info));
 			info = _info;
@@ -2677,9 +2673,7 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 	actors.push_back(serveProcess());
 
 	try {
-		if (connRecord->hasUnresolvedHostnames()) {
-			wait(connRecord->resolveHostnames());
-		}
+		wait(connRecord->resolveHostnames());
 		ServerCoordinators coordinators(connRecord);
 		if (g_network->isSimulated()) {
 			whitelistBinPaths = ",, random_path,  /bin/snap_create.sh,,";
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index 9ca399fa1c..6b8cfe1553 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -930,9 +930,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				boost::split(
 				    process_addresses, coordinator_processes_key.get().toString(), [](char c) { return c == ','; });
 				ASSERT(process_addresses.size() == cs.coordinators().size() + cs.hostnames.size());
-				if (cs.hasUnresolvedHostnames) {
-					wait(cs.resolveHostnames());
-				}
+				wait(cs.resolveHostnames());
 				// compare the coordinator process network addresses one by one
 				for (const auto& network_address : cs.coordinators()) {
 					ASSERT(std::find(process_addresses.begin(), process_addresses.end(), network_address.toString()) !=
@@ -1052,9 +1050,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					Optional<Value> res = wait(tx->get(coordinatorsKey));
 					ASSERT(res.present()); // Otherwise, database is in a bad state
 					state ClusterConnectionString csNew(res.get().toString());
-					if (csNew.hasUnresolvedHostnames) {
-						wait(csNew.resolveHostnames());
-					}
+					wait(csNew.resolveHostnames());
 					ASSERT(csNew.coordinators().size() == old_coordinators_processes.size() + 1);
 					// verify the coordinators' addresses
 					for (const auto& network_address : csNew.coordinators()) {

From ea8b2bd3146d0af243d0e50a071443cbeb2674cd Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Tue, 22 Feb 2022 11:15:44 -0800
Subject: [PATCH 100/210] Resolve hostnames synchronously when calling actor is
 impossible.

---
 fdbclient/AutoPublicAddress.cpp |  1 +
 fdbserver/fdbserver.actor.cpp   | 49 ++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/fdbclient/AutoPublicAddress.cpp b/fdbclient/AutoPublicAddress.cpp
index 12eaf78a39..85ea4fb789 100644
--- a/fdbclient/AutoPublicAddress.cpp
+++ b/fdbclient/AutoPublicAddress.cpp
@@ -35,6 +35,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString& ccs) {
 		io_service ioService;
 		ip::udp::socket socket(ioService);
 
+		ccs.resolveHostnamesBlocking();
 		const auto& coordAddr = ccs.coordinators()[0];
 		const auto boostIp = coordAddr.ip.isV6() ? ip::address(ip::address_v6(coordAddr.ip.toV6()))
 		                                         : ip::address(ip::address_v4(coordAddr.ip.toV4()));
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 4f1f32eb5e..0f8576423a 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -833,8 +833,9 @@ std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
 	NetworkAddressList publicNetworkAddresses;
 	NetworkAddressList listenNetworkAddresses;
 
+	connectionRecord.resolveHostnamesBlocking();
 	auto& coordinators = connectionRecord.getConnectionString().coordinators();
-	ASSERT(coordinators.size() + connectionRecord.getConnectionString().hostnames.size() > 0);
+	ASSERT(coordinators.size() > 0);
 
 	for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
 		const std::string& publicAddressStr = publicAddressStrs[ii];
@@ -1022,6 +1023,28 @@ struct CLIOptions {
 		return opts;
 	}
 
+	void buildNetwork(const char* name) {
+		try {
+			if (!publicAddressStrs.empty()) {
+				std::tie(publicAddresses, listenAddresses) =
+				    buildNetworkAddresses(*connectionFile, publicAddressStrs, listenAddressStrs);
+			}
+		} catch (Error&) {
+			printHelpTeaser(name);
+			flushAndExit(FDB_EXIT_ERROR);
+		}
+
+		if (role == ServerRole::ConsistencyCheck) {
+			if (!publicAddressStrs.empty()) {
+				fprintf(stderr, "ERROR: Public address cannot be specified for consistency check processes\n");
+				printHelpTeaser(name);
+				flushAndExit(FDB_EXIT_ERROR);
+			}
+			auto publicIP = determinePublicIPAutomatically(connectionFile->getConnectionString());
+			publicAddresses.address = NetworkAddress(publicIP, ::getpid());
+		}
+	}
+
 private:
 	CLIOptions() = default;
 
@@ -1594,26 +1617,6 @@ private:
 			// failmon?
 		}
 
-		try {
-			if (!publicAddressStrs.empty()) {
-				std::tie(publicAddresses, listenAddresses) =
-				    buildNetworkAddresses(*connectionFile, publicAddressStrs, listenAddressStrs);
-			}
-		} catch (Error&) {
-			printHelpTeaser(argv[0]);
-			flushAndExit(FDB_EXIT_ERROR);
-		}
-
-		if (role == ServerRole::ConsistencyCheck) {
-			if (!publicAddressStrs.empty()) {
-				fprintf(stderr, "ERROR: Public address cannot be specified for consistency check processes\n");
-				printHelpTeaser(argv[0]);
-				flushAndExit(FDB_EXIT_ERROR);
-			}
-			auto publicIP = determinePublicIPAutomatically(connectionFile->getConnectionString());
-			publicAddresses.address = NetworkAddress(publicIP, ::getpid());
-		}
-
 		if (role == ServerRole::Simulation) {
 			Optional<bool> buggifyOverride = checkBuggifyOverride(testFile);
 			if (buggifyOverride.present())
@@ -1692,7 +1695,7 @@ int main(int argc, char* argv[]) {
 		//_set_output_format(_TWO_DIGIT_EXPONENT);
 #endif
 
-		const auto opts = CLIOptions::parseArgs(argc, argv);
+		auto opts = CLIOptions::parseArgs(argc, argv);
 		const auto role = opts.role;
 
 #ifdef _WIN32
@@ -1787,6 +1790,7 @@ int main(int argc, char* argv[]) {
 
 		if (role == ServerRole::Simulation || role == ServerRole::CreateTemplateDatabase) {
 			// startOldSimulator();
+			opts.buildNetwork(argv[0]);
 			startNewSimulator(opts.printSimTime);
 			openTraceFile(NetworkAddress(), opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
 			openTracer(TracerType(deterministicRandom()->randomInt(static_cast<int>(TracerType::DISABLED),
@@ -1795,6 +1799,7 @@ int main(int argc, char* argv[]) {
 			g_network = newNet2(opts.tlsConfig, opts.useThreadPool, true);
 			g_network->addStopCallback(Net2FileSystem::stop);
 			FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT);
+			opts.buildNetwork(argv[0]);
 
 			const bool expectsPublicAddress =
 			    (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::Restore);

From 3c1394578b227a0e7d308ebc37f2a727da43eb85 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Tue, 22 Feb 2022 14:44:08 -0800
Subject: [PATCH 101/210] Address comments.

---
 fdbclient/AutoPublicAddress.cpp   | 1 +
 fdbclient/MonitorLeader.actor.cpp | 4 ++++
 fdbserver/fdbserver.actor.cpp     | 1 +
 fdbserver/worker.actor.cpp        | 1 +
 4 files changed, 7 insertions(+)

diff --git a/fdbclient/AutoPublicAddress.cpp b/fdbclient/AutoPublicAddress.cpp
index 85ea4fb789..3e62b622fa 100644
--- a/fdbclient/AutoPublicAddress.cpp
+++ b/fdbclient/AutoPublicAddress.cpp
@@ -28,6 +28,7 @@
 
 #include "fdbclient/CoordinationInterface.h"
 
+// Determine public IP address by calling the first coordinator.
 IPAddress determinePublicIPAutomatically(ClusterConnectionString& ccs) {
 	try {
 		using namespace boost::asio;
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index ded347878e..abfda0aefd 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -717,6 +717,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 				wait(nomineeChange.onTrigger() || allActors);
 			} catch (Error& e) {
 				if (e.code() == error_code_coordinators_changed) {
+					TraceEvent("MonitorLeaderCoordinatorsChanged").suppressFor(1.0);
 					connRecord->getConnectionString().resetToUnresolved();
 					break;
 				} else {
@@ -1047,6 +1048,8 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			index = (index + 1) % addrs.size();
 			if (index == successIndex) {
 				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
+				// When the client fails talking to all coordinators, we throw coordinators_changed() and let the caller
+				// re-resolve the connection string and retry.
 				throw coordinators_changed();
 			}
 		}
@@ -1076,6 +1079,7 @@ ACTOR Future<Void> monitorProxies(
 			}
 		} catch (Error& e) {
 			if (e.code() == error_code_coordinators_changed) {
+				TraceEvent("MonitorProxiesCoordinatorsChanged").suppressFor(1.0);
 				info.intermediateConnRecord->getConnectionString().resetToUnresolved();
 			} else {
 				throw e;
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 0f8576423a..5159ba2243 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -1023,6 +1023,7 @@ struct CLIOptions {
 		return opts;
 	}
 
+	// Determine publicAddresses and listenAddresses by calling buildNetworkAddresses().
 	void buildNetwork(const char* name) {
 		try {
 			if (!publicAddressStrs.empty()) {
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 70f0e90404..2dff414e94 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -2535,6 +2535,7 @@ ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClus
 			info = _info;
 		} catch (Error& e) {
 			if (e.code() == error_code_coordinators_changed) {
+				TraceEvent("MonitorLeaderWithDelayedCandidacyCoordinatorsChanged").suppressFor(1.0);
 				info.intermediateConnRecord->getConnectionString().resetToUnresolved();
 			} else {
 				throw e;

From 344a14b01020f7772f1120b5ca4f60c4754fe238 Mon Sep 17 00:00:00 2001
From: Ata E Husain Bohra <ata.husain@snowflake.com>
Date: Tue, 22 Feb 2022 17:34:29 -0800
Subject: [PATCH 102/210] Update StreamCipher ctx/cipher management to respect
 determinism (#6427)

* Update StreamCipher ctx/cipher management to respect determinism

StreamCipher keeps record of CipherKeys created
(including globalCipherKey) to ensure the sensitive data gets
ZERO-OUT and not recorded as part of FDB process dump. However,
in current code it is maintained as an unordered_set indexed
by the object itself. Approach adds non determinism given
object pointer based indexing scheme.

Patch addresses the concern by updating the recording to use
a map indexed by UID.
---
 flow/StreamCipher.cpp | 35 ++++++++++++++++++++---------------
 flow/StreamCipher.h   |  7 +++++--
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp
index 2af051f24f..92edff9d7a 100644
--- a/flow/StreamCipher.cpp
+++ b/flow/StreamCipher.cpp
@@ -20,12 +20,14 @@
 
 #include "flow/StreamCipher.h"
 #include "flow/Arena.h"
+#include "flow/IRandom.h"
 #include "flow/ITrace.h"
 #include "flow/UnitTest.h"
 #include <memory>
 
-std::unordered_set<EVP_CIPHER_CTX*> StreamCipher::ctxs;
-std::unordered_set<StreamCipherKey*> StreamCipherKey::cipherKeys;
+UID StreamCipherKey::globalKeyId;
+std::unordered_map<UID, EVP_CIPHER_CTX*> StreamCipher::ctxs;
+std::unordered_map<UID, StreamCipherKey*> StreamCipherKey::cipherKeys;
 std::unique_ptr<StreamCipherKey> StreamCipherKey::globalKey;
 
 bool StreamCipherKey::isGlobalKeyPresent() {
@@ -36,8 +38,9 @@ void StreamCipherKey::allocGlobalCipherKey() {
 	if (StreamCipherKey::isGlobalKeyPresent()) {
 		return;
 	}
+	StreamCipherKey::globalKeyId = deterministicRandom()->randomUniqueID();
 	StreamCipherKey::globalKey = std::make_unique<StreamCipherKey>(AES_256_KEY_LENGTH);
-	StreamCipherKey::cipherKeys.insert(StreamCipherKey::globalKey.get());
+	StreamCipherKey::cipherKeys[StreamCipherKey::globalKeyId] = StreamCipherKey::globalKey.get();
 }
 
 void StreamCipherKey::initializeGlobalRandomTestKey() {
@@ -56,8 +59,8 @@ StreamCipherKey const* StreamCipherKey::getGlobalCipherKey() {
 }
 
 void StreamCipherKey::cleanup() noexcept {
-	for (auto cipherKey : cipherKeys) {
-		cipherKey->reset();
+	for (const auto& itr : cipherKeys) {
+		itr.second->reset();
 	}
 }
 
@@ -67,31 +70,33 @@ void StreamCipherKey::initializeKey(uint8_t* data, int len) {
 	memcpy(arr.get(), data, copyLen);
 }
 
-StreamCipherKey::StreamCipherKey(int size) : arr(std::make_unique<uint8_t[]>(size)), keySize(size) {
+StreamCipherKey::StreamCipherKey(int size)
+  : id(deterministicRandom()->randomUniqueID()), arr(std::make_unique<uint8_t[]>(size)), keySize(size) {
 	memset(arr.get(), 0, keySize);
-	cipherKeys.insert(this);
+	cipherKeys[id] = this;
 }
 
 StreamCipherKey::~StreamCipherKey() {
 	reset();
-	cipherKeys.erase(this);
+	cipherKeys.erase(this->id);
 }
 
 StreamCipher::StreamCipher(int keySize)
-  : ctx(EVP_CIPHER_CTX_new()), hmacCtx(HMAC_CTX_new()), cipherKey(std::make_unique<StreamCipherKey>(keySize)) {
-	ctxs.insert(ctx);
+  : id(deterministicRandom()->randomUniqueID()), ctx(EVP_CIPHER_CTX_new()), hmacCtx(HMAC_CTX_new()),
+    cipherKey(std::make_unique<StreamCipherKey>(keySize)) {
+	ctxs[id] = ctx;
 }
 
 StreamCipher::StreamCipher()
-  : ctx(EVP_CIPHER_CTX_new()), hmacCtx(HMAC_CTX_new()),
+  : id(deterministicRandom()->randomUniqueID()), ctx(EVP_CIPHER_CTX_new()), hmacCtx(HMAC_CTX_new()),
     cipherKey(std::make_unique<StreamCipherKey>(AES_256_KEY_LENGTH)) {
-	ctxs.insert(ctx);
+	ctxs[id] = ctx;
 }
 
 StreamCipher::~StreamCipher() {
 	HMAC_CTX_free(hmacCtx);
 	EVP_CIPHER_CTX_free(ctx);
-	ctxs.erase(ctx);
+	ctxs.erase(id);
 }
 
 EVP_CIPHER_CTX* StreamCipher::getCtx() {
@@ -103,8 +108,8 @@ HMAC_CTX* StreamCipher::getHmacCtx() {
 }
 
 void StreamCipher::cleanup() noexcept {
-	for (auto ctx : ctxs) {
-		EVP_CIPHER_CTX_free(ctx);
+	for (auto itr : ctxs) {
+		EVP_CIPHER_CTX_free(itr.second);
 	}
 }
 
diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h
index 494f37e181..2586ad8392 100644
--- a/flow/StreamCipher.h
+++ b/flow/StreamCipher.h
@@ -44,8 +44,10 @@
 // Wrapper class for openssl implementation of AES GCM
 // encryption/decryption
 class StreamCipherKey : NonCopyable {
+	static UID globalKeyId;
 	static std::unique_ptr<StreamCipherKey> globalKey;
-	static std::unordered_set<StreamCipherKey*> cipherKeys;
+	static std::unordered_map<UID, StreamCipherKey*> cipherKeys;
+	UID id;
 	std::unique_ptr<uint8_t[]> arr;
 	int keySize;
 
@@ -67,7 +69,8 @@ public:
 };
 
 class StreamCipher final : NonCopyable {
-	static std::unordered_set<EVP_CIPHER_CTX*> ctxs;
+	UID id;
+	static std::unordered_map<UID, EVP_CIPHER_CTX*> ctxs;
 	EVP_CIPHER_CTX* ctx;
 	HMAC_CTX* hmacCtx;
 	std::unique_ptr<StreamCipherKey> cipherKey;

From 8cb554a006960b3d9d587b834d98dbab6e132a29 Mon Sep 17 00:00:00 2001
From: "Bharadwaj V.R" <bharadwaj.vr@snowflake.com>
Date: Wed, 23 Feb 2022 10:21:34 -0800
Subject: [PATCH 103/210] Fix formatting issues in DDTeamCollection.actor.cpp

---
 fdbserver/DDTeamCollection.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index cf8d9c1708..e3e1c3d724 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -5644,7 +5644,7 @@ public:
 
 		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
 
-    ASSERT(!resTeam.first.present());
+		ASSERT(!resTeam.first.present());
 
 		return Void();
 	}

From 9ab21fe873cb51097a9ab337858bf4bca7a0224e Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 23 Feb 2022 13:49:53 -0500
Subject: [PATCH 104/210] remove unused knob

---
 fdbclient/ClientKnobs.cpp | 1 -
 fdbclient/ClientKnobs.h   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index f962c9f9ba..429117a3e1 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -122,7 +122,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( DEBUG_USE_GRV_CACHE_CHANCE,              -1.0 ); // For 100% chance at 1.0, this means 0.0 is not 0%. We don't want the default to be 0. 
 	init( FORCE_GRV_CACHE_OFF,                    false );
 	init( GRV_CACHE_RK_COOLDOWN,                   60.0 );
-	init( GRV_THROTTLING_THRESHOLD,                   2 );
 	init( GRV_SUSTAINED_THROTTLING_THRESHOLD,       0.1 );
 
 	// TaskBucket
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 821f8e4935..3100de7053 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -125,7 +125,6 @@ public:
 	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
 	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
 	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
-	int GRV_THROTTLING_THRESHOLD; // Number of iterations spent in the request queue loop which is considered throttling
 	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
 	                                           // specified here, ratekeeper is throttling and not a false positive
 

From ffbe10696ae3d9550a777cd410522e8f60b1de0a Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Wed, 23 Feb 2022 17:59:37 -0500
Subject: [PATCH 105/210] updated generated go bindings

---
 bindings/go/src/fdb/generated.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index 2acd3c4a21..8be49f5c79 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -598,6 +598,11 @@ func (o TransactionOptions) SetBypassUnreadable() error {
 	return o.setOpt(1100, nil)
 }
 
+// Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions.
+func (o TransactionOptions) SetUseGrvCache() error {
+	return o.setOpt(1101, nil)
+}
+
 type StreamingMode int
 
 const (

From f14e08a991dd385332e23409658375b7fed78cca Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhewang@Zhes-MacBook-Pro.local>
Date: Thu, 17 Feb 2022 17:04:01 -0500
Subject: [PATCH 106/210] addRocksDBPerfContextMetrics

---
 fdbclient/ServerKnobs.cpp                |   3 +-
 fdbclient/ServerKnobs.h                  |   2 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 355 ++++++++++++++++++++++-
 3 files changed, 349 insertions(+), 11 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index e6b87b864e..7958ae1993 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -364,7 +364,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC,                0 );
 	// If true, enables dynamic adjustment of ROCKSDB_WRITE_RATE_LIMITER_BYTES according to the recent demand of background IO.
 	init( ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE,                 true );
-
+	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
+	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 					  0.0001 );
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index f8202629d7..9f06938334 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -295,6 +295,8 @@ public:
 	bool ROCKSDB_READ_RANGE_REUSE_ITERATORS;
 	int64_t ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC;
 	bool ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE;
+	bool ROCKSDB_PERFCONTEXT_ENABLE; // Enable rocks perf context metrics. May cause performance overhead
+	double ROCKSDB_PERFCONTEXT_SAMPLE_RATE;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 21bc074231..090256df0e 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -11,6 +11,8 @@
 #include <rocksdb/version.h>
 #include <rocksdb/utilities/table_properties_collectors.h>
 #include <rocksdb/rate_limiter.h>
+#include <rocksdb/perf_context.h>
+#include <rocksdb/c.h>
 #if defined __has_include
 #if __has_include(<liburing.h>)
 #include <liburing.h>
@@ -312,6 +314,271 @@ private:
 	uint64_t iteratorsReuseCount;
 };
 
+class PerfContextMetrics {
+public:
+	PerfContextMetrics();
+	void reset();
+	void set(int index);
+	void log(bool ignoreZeroMetric);
+
+private:
+	std::vector<std::tuple<const char*, int, std::vector<uint64_t>>> metrics;
+	uint64_t getRocksdbPerfcontextMetric(int metric);
+};
+
+PerfContextMetrics::PerfContextMetrics() {
+	metrics = {
+		{ "UserKeyComparisonCount", rocksdb_user_key_comparison_count, {} },
+		{ "BlockCacheHitCount", rocksdb_block_cache_hit_count, {} },
+		{ "BlockReadCount", rocksdb_block_read_count, {} },
+		{ "BlockReadByte", rocksdb_block_read_byte, {} },
+		{ "BlockReadTime", rocksdb_block_read_time, {} },
+		{ "BlockChecksumTime", rocksdb_block_checksum_time, {} },
+		{ "BlockDecompressTime", rocksdb_block_decompress_time, {} },
+		{ "GetReadBytes", rocksdb_get_read_bytes, {} },
+		{ "MultigetReadBytes", rocksdb_multiget_read_bytes, {} },
+		{ "IterReadBytes", rocksdb_iter_read_bytes, {} },
+		{ "InternalKeySkippedCount", rocksdb_internal_key_skipped_count, {} },
+		{ "InternalDeleteSkippedCount", rocksdb_internal_delete_skipped_count, {} },
+		{ "InternalRecentSkippedCount", rocksdb_internal_recent_skipped_count, {} },
+		{ "InternalMergeCount", rocksdb_internal_merge_count, {} },
+		{ "GetSnapshotTime", rocksdb_get_snapshot_time, {} },
+		{ "GetFromMemtableTime", rocksdb_get_from_memtable_time, {} },
+		{ "GetFromMemtableCount", rocksdb_get_from_memtable_count, {} },
+		{ "GetPostProcessTime", rocksdb_get_post_process_time, {} },
+		{ "GetFromOutputFilesTime", rocksdb_get_from_output_files_time, {} },
+		{ "SeekOnMemtableTime", rocksdb_seek_on_memtable_time, {} },
+		{ "SeekOnMemtableCount", rocksdb_seek_on_memtable_count, {} },
+		{ "NextOnMemtableCount", rocksdb_next_on_memtable_count, {} },
+		{ "PrevOnMemtableCount", rocksdb_prev_on_memtable_count, {} },
+		{ "SeekChildSeekTime", rocksdb_seek_child_seek_time, {} },
+		{ "SeekChildSeekCount", rocksdb_seek_child_seek_count, {} },
+		{ "SeekMinHeapTime", rocksdb_seek_min_heap_time, {} },
+		{ "SeekMaxHeapTime", rocksdb_seek_max_heap_time, {} },
+		{ "SeekInternalSeekTime", rocksdb_seek_internal_seek_time, {} },
+		{ "FindNextUserEntryTime", rocksdb_find_next_user_entry_time, {} },
+		{ "WriteWalTime", rocksdb_write_wal_time, {} },
+		{ "WriteMemtableTime", rocksdb_write_memtable_time, {} },
+		{ "WriteDelayTime", rocksdb_write_delay_time, {} },
+		{ "WritePreAndPostProcessTime", rocksdb_write_pre_and_post_process_time, {} },
+		{ "DbMutexLockNanos", rocksdb_db_mutex_lock_nanos, {} },
+		{ "DbConditionWaitNanos", rocksdb_db_condition_wait_nanos, {} },
+		{ "MergeOperatorTimeNanos", rocksdb_merge_operator_time_nanos, {} },
+		{ "ReadIndexBlockNanos", rocksdb_read_index_block_nanos, {} },
+		{ "ReadFilterBlockNanos", rocksdb_read_filter_block_nanos, {} },
+		{ "NewTableBlockIterNanos", rocksdb_new_table_block_iter_nanos, {} },
+		{ "NewTableIteratorNanos", rocksdb_new_table_iterator_nanos, {} },
+		{ "BlockSeekNanos", rocksdb_block_seek_nanos, {} },
+		{ "FindTableNanos", rocksdb_find_table_nanos, {} },
+		{ "BloomMemtableHitCount", rocksdb_bloom_memtable_hit_count, {} },
+		{ "BloomMemtableMissCount", rocksdb_bloom_memtable_miss_count, {} },
+		{ "BloomSstHitCount", rocksdb_bloom_sst_hit_count, {} },
+		{ "BloomSstMissCount", rocksdb_bloom_sst_miss_count, {} },
+		{ "KeyLockWaitTime", rocksdb_key_lock_wait_time, {} },
+		{ "KeyLockWaitCount", rocksdb_key_lock_wait_count, {} },
+		{ "EnvNewSequentialFileNanos", rocksdb_env_new_sequential_file_nanos, {} },
+		{ "EnvNewRandomAccessFileNanos", rocksdb_env_new_random_access_file_nanos, {} },
+		{ "EnvNewWritableFileNanos", rocksdb_env_new_writable_file_nanos, {} },
+		{ "EnvReuseWritableFileNanos", rocksdb_env_reuse_writable_file_nanos, {} },
+		{ "EnvNewRandomRwFileNanos", rocksdb_env_new_random_rw_file_nanos, {} },
+		{ "EnvNewDirectoryNanos", rocksdb_env_new_directory_nanos, {} },
+		{ "EnvFileExistsNanos", rocksdb_env_file_exists_nanos, {} },
+		{ "EnvGetChildrenNanos", rocksdb_env_get_children_nanos, {} },
+		{ "EnvGetChildrenFileAttributesNanos", rocksdb_env_get_children_file_attributes_nanos, {} },
+		{ "EnvDeleteFileNanos", rocksdb_env_delete_file_nanos, {} },
+		{ "EnvCreateDirNanos", rocksdb_env_create_dir_nanos, {} },
+		{ "EnvCreateDirIfMissingNanos", rocksdb_env_create_dir_if_missing_nanos, {} },
+		{ "EnvDeleteDirNanos", rocksdb_env_delete_dir_nanos, {} },
+		{ "EnvGetFileSizeNanos", rocksdb_env_get_file_size_nanos, {} },
+		{ "EnvGetFileModificationTimeNanos", rocksdb_env_get_file_modification_time_nanos, {} },
+		{ "EnvRenameFileNanos", rocksdb_env_rename_file_nanos, {} },
+		{ "EnvLinkFileNanos", rocksdb_env_link_file_nanos, {} },
+		{ "EnvLockFileNanos", rocksdb_env_lock_file_nanos, {} },
+		{ "EnvUnlockFileNanos", rocksdb_env_unlock_file_nanos, {} },
+		{ "EnvNewLoggerNanos", rocksdb_env_new_logger_nanos, {} },
+	};
+	for (auto& [name, metric, vals] : metrics) { // readers, then writer
+		for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
+			vals.push_back(0); // add reader
+		}
+		vals.push_back(0); // add writer
+	}
+}
+
+void PerfContextMetrics::reset() {
+	rocksdb::get_perf_context()->Reset();
+}
+
+void PerfContextMetrics::set(int index) {
+	for (auto& [name, metric, vals] : metrics) {
+		vals[index] = getRocksdbPerfcontextMetric(metric);
+	}
+}
+
+void PerfContextMetrics::log(bool ignoreZeroMetric) {
+	TraceEvent e("RocksDBPerfContextMetrics");
+	e.setMaxEventLength(20000);
+	for (auto& [name, metric, vals] : metrics) {
+		uint64_t s = 0;
+		for (auto& v : vals) {
+			s = s + v;
+		}
+		if (ignoreZeroMetric && s == 0)
+			continue;
+		e.detail("Sum" + (std::string)name, s);
+		for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
+			if (vals[i] != 0)
+				e.detail("RD" + std::to_string(i) + name, vals[i]);
+		}
+		if (vals[SERVER_KNOBS->ROCKSDB_READ_PARALLELISM] != 0)
+			e.detail("WR" + (std::string)name, vals[SERVER_KNOBS->ROCKSDB_READ_PARALLELISM]);
+	}
+}
+
+uint64_t PerfContextMetrics::getRocksdbPerfcontextMetric(int metric) {
+	switch (metric) {
+	case rocksdb_user_key_comparison_count:
+		return rocksdb::get_perf_context()->user_key_comparison_count;
+	case rocksdb_block_cache_hit_count:
+		return rocksdb::get_perf_context()->block_cache_hit_count;
+	case rocksdb_block_read_count:
+		return rocksdb::get_perf_context()->block_read_count;
+	case rocksdb_block_read_byte:
+		return rocksdb::get_perf_context()->block_read_byte;
+	case rocksdb_block_read_time:
+		return rocksdb::get_perf_context()->block_read_time;
+	case rocksdb_block_checksum_time:
+		return rocksdb::get_perf_context()->block_checksum_time;
+	case rocksdb_block_decompress_time:
+		return rocksdb::get_perf_context()->block_decompress_time;
+	case rocksdb_get_read_bytes:
+		return rocksdb::get_perf_context()->get_read_bytes;
+	case rocksdb_multiget_read_bytes:
+		return rocksdb::get_perf_context()->multiget_read_bytes;
+	case rocksdb_iter_read_bytes:
+		return rocksdb::get_perf_context()->iter_read_bytes;
+	case rocksdb_internal_key_skipped_count:
+		return rocksdb::get_perf_context()->internal_key_skipped_count;
+	case rocksdb_internal_delete_skipped_count:
+		return rocksdb::get_perf_context()->internal_delete_skipped_count;
+	case rocksdb_internal_recent_skipped_count:
+		return rocksdb::get_perf_context()->internal_recent_skipped_count;
+	case rocksdb_internal_merge_count:
+		return rocksdb::get_perf_context()->internal_merge_count;
+	case rocksdb_get_snapshot_time:
+		return rocksdb::get_perf_context()->get_snapshot_time;
+	case rocksdb_get_from_memtable_time:
+		return rocksdb::get_perf_context()->get_from_memtable_time;
+	case rocksdb_get_from_memtable_count:
+		return rocksdb::get_perf_context()->get_from_memtable_count;
+	case rocksdb_get_post_process_time:
+		return rocksdb::get_perf_context()->get_post_process_time;
+	case rocksdb_get_from_output_files_time:
+		return rocksdb::get_perf_context()->get_from_output_files_time;
+	case rocksdb_seek_on_memtable_time:
+		return rocksdb::get_perf_context()->seek_on_memtable_time;
+	case rocksdb_seek_on_memtable_count:
+		return rocksdb::get_perf_context()->seek_on_memtable_count;
+	case rocksdb_next_on_memtable_count:
+		return rocksdb::get_perf_context()->next_on_memtable_count;
+	case rocksdb_prev_on_memtable_count:
+		return rocksdb::get_perf_context()->prev_on_memtable_count;
+	case rocksdb_seek_child_seek_time:
+		return rocksdb::get_perf_context()->seek_child_seek_time;
+	case rocksdb_seek_child_seek_count:
+		return rocksdb::get_perf_context()->seek_child_seek_count;
+	case rocksdb_seek_min_heap_time:
+		return rocksdb::get_perf_context()->seek_min_heap_time;
+	case rocksdb_seek_max_heap_time:
+		return rocksdb::get_perf_context()->seek_max_heap_time;
+	case rocksdb_seek_internal_seek_time:
+		return rocksdb::get_perf_context()->seek_internal_seek_time;
+	case rocksdb_find_next_user_entry_time:
+		return rocksdb::get_perf_context()->find_next_user_entry_time;
+	case rocksdb_write_wal_time:
+		return rocksdb::get_perf_context()->write_wal_time;
+	case rocksdb_write_memtable_time:
+		return rocksdb::get_perf_context()->write_memtable_time;
+	case rocksdb_write_delay_time:
+		return rocksdb::get_perf_context()->write_delay_time;
+	case rocksdb_write_pre_and_post_process_time:
+		return rocksdb::get_perf_context()->write_pre_and_post_process_time;
+	case rocksdb_db_mutex_lock_nanos:
+		return rocksdb::get_perf_context()->db_mutex_lock_nanos;
+	case rocksdb_db_condition_wait_nanos:
+		return rocksdb::get_perf_context()->db_condition_wait_nanos;
+	case rocksdb_merge_operator_time_nanos:
+		return rocksdb::get_perf_context()->merge_operator_time_nanos;
+	case rocksdb_read_index_block_nanos:
+		return rocksdb::get_perf_context()->read_index_block_nanos;
+	case rocksdb_read_filter_block_nanos:
+		return rocksdb::get_perf_context()->read_filter_block_nanos;
+	case rocksdb_new_table_block_iter_nanos:
+		return rocksdb::get_perf_context()->new_table_block_iter_nanos;
+	case rocksdb_new_table_iterator_nanos:
+		return rocksdb::get_perf_context()->new_table_iterator_nanos;
+	case rocksdb_block_seek_nanos:
+		return rocksdb::get_perf_context()->block_seek_nanos;
+	case rocksdb_find_table_nanos:
+		return rocksdb::get_perf_context()->find_table_nanos;
+	case rocksdb_bloom_memtable_hit_count:
+		return rocksdb::get_perf_context()->bloom_memtable_hit_count;
+	case rocksdb_bloom_memtable_miss_count:
+		return rocksdb::get_perf_context()->bloom_memtable_miss_count;
+	case rocksdb_bloom_sst_hit_count:
+		return rocksdb::get_perf_context()->bloom_sst_hit_count;
+	case rocksdb_bloom_sst_miss_count:
+		return rocksdb::get_perf_context()->bloom_sst_miss_count;
+	case rocksdb_key_lock_wait_time:
+		return rocksdb::get_perf_context()->key_lock_wait_time;
+	case rocksdb_key_lock_wait_count:
+		return rocksdb::get_perf_context()->key_lock_wait_count;
+	case rocksdb_env_new_sequential_file_nanos:
+		return rocksdb::get_perf_context()->env_new_sequential_file_nanos;
+	case rocksdb_env_new_random_access_file_nanos:
+		return rocksdb::get_perf_context()->env_new_random_access_file_nanos;
+	case rocksdb_env_new_writable_file_nanos:
+		return rocksdb::get_perf_context()->env_new_writable_file_nanos;
+	case rocksdb_env_reuse_writable_file_nanos:
+		return rocksdb::get_perf_context()->env_reuse_writable_file_nanos;
+	case rocksdb_env_new_random_rw_file_nanos:
+		return rocksdb::get_perf_context()->env_new_random_rw_file_nanos;
+	case rocksdb_env_new_directory_nanos:
+		return rocksdb::get_perf_context()->env_new_directory_nanos;
+	case rocksdb_env_file_exists_nanos:
+		return rocksdb::get_perf_context()->env_file_exists_nanos;
+	case rocksdb_env_get_children_nanos:
+		return rocksdb::get_perf_context()->env_get_children_nanos;
+	case rocksdb_env_get_children_file_attributes_nanos:
+		return rocksdb::get_perf_context()->env_get_children_file_attributes_nanos;
+	case rocksdb_env_delete_file_nanos:
+		return rocksdb::get_perf_context()->env_delete_file_nanos;
+	case rocksdb_env_create_dir_nanos:
+		return rocksdb::get_perf_context()->env_create_dir_nanos;
+	case rocksdb_env_create_dir_if_missing_nanos:
+		return rocksdb::get_perf_context()->env_create_dir_if_missing_nanos;
+	case rocksdb_env_delete_dir_nanos:
+		return rocksdb::get_perf_context()->env_delete_dir_nanos;
+	case rocksdb_env_get_file_size_nanos:
+		return rocksdb::get_perf_context()->env_get_file_size_nanos;
+	case rocksdb_env_get_file_modification_time_nanos:
+		return rocksdb::get_perf_context()->env_get_file_modification_time_nanos;
+	case rocksdb_env_rename_file_nanos:
+		return rocksdb::get_perf_context()->env_rename_file_nanos;
+	case rocksdb_env_link_file_nanos:
+		return rocksdb::get_perf_context()->env_link_file_nanos;
+	case rocksdb_env_lock_file_nanos:
+		return rocksdb::get_perf_context()->env_lock_file_nanos;
+	case rocksdb_env_unlock_file_nanos:
+		return rocksdb::get_perf_context()->env_unlock_file_nanos;
+	case rocksdb_env_new_logger_nanos:
+		return rocksdb::get_perf_context()->env_new_logger_nanos;
+	default:
+		break;
+	}
+	return 0;
+}
+
 ACTOR Future<Void> refreshReadIteratorPool(std::shared_ptr<ReadIteratorPool> readIterPool) {
 	if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
 		loop {
@@ -336,6 +603,7 @@ ACTOR Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetc
 }
 
 ACTOR Future<Void> rocksDBMetricLogger(std::shared_ptr<rocksdb::Statistics> statistics,
+                                       std::shared_ptr<PerfContextMetrics> perfContextMetrics,
                                        rocksdb::DB* db,
                                        std::shared_ptr<ReadIteratorPool> readIterPool) {
 	state std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats = {
@@ -431,6 +699,10 @@ ACTOR Future<Void> rocksDBMetricLogger(std::shared_ptr<rocksdb::Statistics> stat
 		stat = readIterPool->numTimesReadIteratorsReused();
 		e.detail("NumTimesReadIteratorsReused", stat - readIteratorPoolStats["NumTimesReadIteratorsReused"]);
 		readIteratorPoolStats["NumTimesReadIteratorsReused"] = stat;
+
+		if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE) {
+			perfContextMetrics->log(true);
+		}
 	}
 }
 
@@ -458,6 +730,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	struct Writer : IThreadPoolReceiver {
 		DB& db;
+
 		UID id;
 		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
 		Reference<Histogram> commitLatencyHistogram;
@@ -466,9 +739,16 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		Reference<Histogram> writeHistogram;
 		Reference<Histogram> deleteCompactRangeHistogram;
 		std::shared_ptr<ReadIteratorPool> readIterPool;
+		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+		int threadIndex;
 
-		explicit Writer(DB& db, UID id, std::shared_ptr<ReadIteratorPool> readIterPool)
-		  : db(db), id(id), readIterPool(readIterPool),
+		explicit Writer(DB& db,
+		                UID id,
+		                std::shared_ptr<ReadIteratorPool> readIterPool,
+		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
+		                int threadIndex)
+		  : db(db), id(id), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
+		    threadIndex(threadIndex),
 		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
 		                    ? rocksdb::NewGenericRateLimiter(
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
@@ -491,7 +771,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		                                           Histogram::Unit::microseconds)),
 		    deleteCompactRangeHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
 		                                                        ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM,
-		                                                        Histogram::Unit::microseconds)) {}
+		                                                        Histogram::Unit::microseconds)) {
+			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE) {
+				// Enable perf context on the same thread with the db thread
+				rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+				perfContextMetrics->reset();
+			}
+		}
 
 		~Writer() override {
 			if (db) {
@@ -542,11 +828,11 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					// The current thread and main thread are same when the code runs in simulation.
 					// blockUntilReady() is getting the thread into deadlock state, so directly calling
 					// the metricsLogger.
-					a.metrics = rocksDBMetricLogger(options.statistics, db, readIterPool) &&
+					a.metrics = rocksDBMetricLogger(options.statistics, perfContextMetrics, db, readIterPool) &&
 					            flowLockLogger(a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool);
 				} else {
 					onMainThread([&] {
-						a.metrics = rocksDBMetricLogger(options.statistics, db, readIterPool) &&
+						a.metrics = rocksDBMetricLogger(options.statistics, perfContextMetrics, db, readIterPool) &&
 						            flowLockLogger(a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool);
 						return Future<bool>(true);
 					}).blockUntilReady();
@@ -586,6 +872,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 		};
 		void action(CommitAction& a) {
+			bool doPerfContextMetrics =
+			    SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE &&
+			    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE);
+			if (doPerfContextMetrics) {
+				perfContextMetrics->reset();
+			}
 			double commitBeginTime;
 			if (a.getHistograms) {
 				commitBeginTime = timer_monotonic();
@@ -632,6 +924,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				commitActionHistogram->sampleSeconds(currTime - commitBeginTime);
 				commitLatencyHistogram->sampleSeconds(currTime - a.startTime);
 			}
+			if (doPerfContextMetrics) {
+				perfContextMetrics->set(threadIndex);
+			}
 		}
 
 		struct CloseAction : TypedAction<Writer, CloseAction> {
@@ -684,9 +979,14 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		Reference<Histogram> readValueGetHistogram;
 		Reference<Histogram> readPrefixGetHistogram;
 		std::shared_ptr<ReadIteratorPool> readIterPool;
+		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+		int threadIndex;
 
-		explicit Reader(DB& db, std::shared_ptr<ReadIteratorPool> readIterPool)
-		  : db(db), readIterPool(readIterPool),
+		explicit Reader(DB& db,
+		                std::shared_ptr<ReadIteratorPool> readIterPool,
+		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
+		                int threadIndex)
+		  : db(db), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics), threadIndex(threadIndex),
 		    readRangeLatencyHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
 		                                                      ROCKSDB_READRANGE_LATENCY_HISTOGRAM,
 		                                                      Histogram::Unit::microseconds)),
@@ -734,6 +1034,11 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT;
 				readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
 			}
+			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE) {
+				// Enable perf context on the same thread with the db thread
+				rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+				perfContextMetrics->reset();
+			}
 		}
 
 		void init() override {}
@@ -752,6 +1057,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
 		void action(ReadValueAction& a) {
+			bool doPerfContextMetrics =
+			    SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE &&
+			    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE);
+			if (doPerfContextMetrics) {
+				perfContextMetrics->reset();
+			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				readValueQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
@@ -801,6 +1112,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readValueActionHistogram->sampleSeconds(currTime - readBeginTime);
 				readValueLatencyHistogram->sampleSeconds(currTime - a.startTime);
 			}
+			if (doPerfContextMetrics) {
+				perfContextMetrics->set(threadIndex);
+			}
 		}
 
 		struct ReadValuePrefixAction : TypedAction<Reader, ReadValuePrefixAction> {
@@ -818,6 +1132,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
 		void action(ReadValuePrefixAction& a) {
+			bool doPerfContextMetrics =
+			    SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE &&
+			    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE);
+			if (doPerfContextMetrics) {
+				perfContextMetrics->reset();
+			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				readPrefixQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
@@ -871,6 +1191,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readPrefixActionHistogram->sampleSeconds(currTime - readBeginTime);
 				readPrefixLatencyHistogram->sampleSeconds(currTime - a.startTime);
 			}
+			if (doPerfContextMetrics) {
+				perfContextMetrics->set(threadIndex);
+			}
 		}
 
 		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
@@ -887,6 +1210,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
 		};
 		void action(ReadRangeAction& a) {
+			bool doPerfContextMetrics =
+			    SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE &&
+			    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE);
+			if (doPerfContextMetrics) {
+				perfContextMetrics->reset();
+			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				readRangeQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
@@ -983,10 +1312,14 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readRangeActionHistogram->sampleSeconds(currTime - readBeginTime);
 				readRangeLatencyHistogram->sampleSeconds(currTime - a.startTime);
 			}
+			if (doPerfContextMetrics) {
+				perfContextMetrics->set(threadIndex);
+			}
 		}
 	};
 
 	DB db = nullptr;
+	std::shared_ptr<PerfContextMetrics> perfContextMetrics;
 	std::string path;
 	UID id;
 	Reference<IThreadPool> writeThread;
@@ -1015,7 +1348,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	Counters counters;
 
 	explicit RocksDBKeyValueStore(const std::string& path, UID id)
-	  : path(path), id(id), readIterPool(new ReadIteratorPool(db, path)),
+	  : path(path), id(id), perfContextMetrics(new PerfContextMetrics()), readIterPool(new ReadIteratorPool(db, path)),
 	    readSemaphore(SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
 	    fetchSemaphore(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
 	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
@@ -1038,10 +1371,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			writeThread = createGenericThreadPool();
 			readThreads = createGenericThreadPool();
 		}
-		writeThread->addThread(new Writer(db, id, readIterPool), "fdb-rocksdb-wr");
+		writeThread->addThread(
+		    new Writer(db, id, readIterPool, perfContextMetrics, SERVER_KNOBS->ROCKSDB_READ_PARALLELISM),
+		    "fdb-rocksdb-wr");
 		TraceEvent("RocksDBReadThreads").detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
 		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
-			readThreads->addThread(new Reader(db, readIterPool), "fdb-rocksdb-re");
+			readThreads->addThread(new Reader(db, readIterPool, perfContextMetrics, i), "fdb-rocksdb-re");
 		}
 	}
 

From f379cf4d05920d00a50501298f02fc5ff429844e Mon Sep 17 00:00:00 2001
From: Leonidas Tsampros <ltsampros@apple.com>
Date: Tue, 22 Feb 2022 13:42:56 +0000
Subject: [PATCH 107/210] Use full version of fdb to support incompatible
 release candidates

---
 packaging/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index 7f316ff271..f647453209 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -96,13 +96,13 @@ RUN curl -Ls $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz |
 
 # Install additional FoundationDB Client Libraries
 RUN for version in $FDB_LIBRARY_VERSIONS; do \
-    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \
+    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version}.so; \
     done
 
 # Install additional FoundationDB Client Libraries (for sidecar)
 RUN mkdir -p /var/fdb/lib && \
     for version in $FDB_LIBRARY_VERSIONS; do \
-    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \
+    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version}.so; \
     done
 
 RUN curl -Ls $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so

From 4ff60db82b451def2b25135f88aeba79337196dd Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Thu, 24 Feb 2022 11:55:48 -0600
Subject: [PATCH 108/210] Revert "Use full version of fdb to support
 incompatible release candidates"

This reverts commit f379cf4d05920d00a50501298f02fc5ff429844e.
---
 packaging/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index f647453209..7f316ff271 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -96,13 +96,13 @@ RUN curl -Ls $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz |
 
 # Install additional FoundationDB Client Libraries
 RUN for version in $FDB_LIBRARY_VERSIONS; do \
-    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version}.so; \
+    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \
     done
 
 # Install additional FoundationDB Client Libraries (for sidecar)
 RUN mkdir -p /var/fdb/lib && \
     for version in $FDB_LIBRARY_VERSIONS; do \
-    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version}.so; \
+    curl -Ls $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \
     done
 
 RUN curl -Ls $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so

From 250a88e682f589099a8910c562181a347b38e838 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Thu, 24 Feb 2022 12:25:52 -0800
Subject: [PATCH 109/210] Enforce that trace event suppression calls happen
 first when using trace event call chaining. Fix various instances where we
 weren't following this requirement.

---
 fdbbackup/backup.actor.cpp                    |  6 +-
 fdbcli/IncludeCommand.actor.cpp               |  4 +-
 fdbcli/fdbcli.actor.cpp                       |  4 +-
 fdbclient/BackupContainer.actor.cpp           |  5 +-
 fdbclient/BackupContainerFileSystem.actor.cpp | 10 +-
 fdbclient/ConfigKnobs.cpp                     |  2 +-
 fdbclient/DatabaseBackupAgent.actor.cpp       |  4 +-
 fdbclient/FileBackupAgent.actor.cpp           | 12 +--
 fdbclient/HTTP.actor.cpp                      |  6 +-
 fdbclient/ManagementAPI.actor.cpp             | 18 ++--
 fdbclient/MonitorLeader.actor.cpp             |  4 +-
 fdbclient/MultiVersionTransaction.actor.cpp   | 24 ++---
 fdbclient/NativeAPI.actor.cpp                 |  8 +-
 fdbclient/ReadYourWrites.actor.cpp            |  2 +-
 fdbclient/S3BlobStore.actor.cpp               |  8 +-
 fdbrpc/AsyncFileNonDurable.actor.h            |  2 +-
 fdbrpc/FlowTransport.actor.cpp                | 16 ++--
 fdbrpc/sim2.actor.cpp                         |  2 +-
 fdbserver/BackupWorker.actor.cpp              |  4 +-
 fdbserver/BlobManager.actor.cpp               |  8 +-
 fdbserver/BlobWorker.actor.cpp                |  4 +-
 fdbserver/ClusterController.actor.cpp         | 16 ++--
 fdbserver/ClusterController.actor.h           |  4 +-
 fdbserver/CommitProxyServer.actor.cpp         | 10 +-
 fdbserver/Coordination.actor.cpp              |  2 +-
 fdbserver/CoroFlow.actor.cpp                  |  4 +-
 fdbserver/CoroFlowCoro.actor.cpp              |  6 +-
 fdbserver/DDTeamCollection.actor.cpp          |  4 +-
 fdbserver/DataDistribution.actor.cpp          | 16 ++--
 fdbserver/DataDistributionQueue.actor.cpp     | 14 +--
 fdbserver/DiskQueue.actor.cpp                 | 10 +-
 fdbserver/EncryptKeyProxy.actor.cpp           |  2 +-
 fdbserver/GrvProxyServer.actor.cpp            |  2 +-
 fdbserver/KeyValueStoreMemory.actor.cpp       |  2 +-
 fdbserver/KeyValueStoreSQLite.actor.cpp       |  2 +-
 fdbserver/LogRouter.actor.cpp                 |  6 +-
 fdbserver/LogSystemPeekCursor.actor.cpp       |  2 +-
 fdbserver/MoveKeys.actor.cpp                  |  4 +-
 fdbserver/MutationTracking.cpp                |  2 +-
 fdbserver/OldTLogServer_4_6.actor.cpp         |  6 +-
 fdbserver/OldTLogServer_6_0.actor.cpp         | 10 +-
 fdbserver/OldTLogServer_6_2.actor.cpp         | 10 +-
 fdbserver/QuietDatabase.actor.cpp             |  2 +-
 fdbserver/Ratekeeper.actor.cpp                |  6 +-
 fdbserver/Resolver.actor.cpp                  |  2 +-
 fdbserver/RestoreApplier.actor.cpp            | 20 ++--
 fdbserver/RestoreController.actor.cpp         |  2 +-
 fdbserver/RestoreLoader.actor.cpp             | 24 ++---
 fdbserver/RestoreWorker.actor.cpp             |  2 +-
 fdbserver/SimulatedCluster.actor.cpp          |  4 +-
 fdbserver/StorageCache.actor.cpp              |  4 +-
 fdbserver/TLogServer.actor.cpp                | 10 +-
 fdbserver/VFSAsync.cpp                        |  2 +-
 fdbserver/VersionedBTree.actor.cpp            | 22 ++---
 fdbserver/fdbserver.actor.cpp                 |  4 +-
 fdbserver/networktest.actor.cpp               |  8 +-
 fdbserver/storageserver.actor.cpp             |  8 +-
 fdbserver/tester.actor.cpp                    |  2 +-
 fdbserver/worker.actor.cpp                    |  8 +-
 .../workloads/BackupToDBUpgrade.actor.cpp     |  2 +-
 fdbserver/workloads/ConfigIncrement.actor.cpp |  4 +-
 .../workloads/ConsistencyCheck.actor.cpp      |  8 +-
 .../workloads/DiskFailureInjection.actor.cpp  |  4 +-
 .../workloads/FastTriggeredWatches.actor.cpp  |  2 +-
 fdbserver/workloads/LowLatency.actor.cpp      |  4 +-
 fdbserver/workloads/Mako.actor.cpp            |  2 +-
 fdbserver/workloads/Performance.actor.cpp     |  4 +-
 fdbserver/workloads/RandomMoveKeys.actor.cpp  |  2 +-
 .../SpecialKeySpaceCorrectness.actor.cpp      |  4 +-
 fdbserver/workloads/UnitTests.actor.cpp       |  2 +-
 fdbserver/workloads/WriteDuringRead.actor.cpp |  2 +-
 flow/Platform.actor.cpp                       | 20 ++--
 flow/ThreadHelper.actor.h                     |  2 +-
 flow/Trace.cpp                                | 93 +++++++++----------
 flow/Trace.h                                  | 85 +++++++++--------
 flow/flow.h                                   |  2 +-
 flow/genericactors.actor.h                    |  4 +-
 77 files changed, 342 insertions(+), 321 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 38a43eef08..f2a1356c35 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -1690,7 +1690,7 @@ ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
 				readMore = true;
 		} catch (Error& e) {
 			// If doc can't be parsed or isn't alive, delete it.
-			TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key).error(e, true);
+			TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").errorUnsuppressed(e).detail("Key", docs[i].key);
 			tr->clear(docs[i].key);
 			// If limit is 1 then read more.
 			if (limit == 1)
@@ -3887,9 +3887,9 @@ int main(int argc, char* argv[]) {
 				} else {
 					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
+					    .error(e)
 					    .detail("Knob", printable(knobName))
-					    .detail("Value", printable(knobValueString))
-					    .error(e);
+					    .detail("Value", printable(knobValueString));
 					throw;
 				}
 			}
diff --git a/fdbcli/IncludeCommand.actor.cpp b/fdbcli/IncludeCommand.actor.cpp
index fda54f6fcb..2a72a214e2 100644
--- a/fdbcli/IncludeCommand.actor.cpp
+++ b/fdbcli/IncludeCommand.actor.cpp
@@ -59,7 +59,7 @@ ACTOR Future<Void> includeLocalities(Reference<IDatabase> db,
 			wait(safeThreadFutureToFuture(tr->commit()));
 			return Void();
 		} catch (Error& e) {
-			TraceEvent("IncludeLocalitiesError").error(e, true);
+			TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -99,7 +99,7 @@ ACTOR Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressEx
 			wait(safeThreadFutureToFuture(tr->commit()));
 			return Void();
 		} catch (Error& e) {
-			TraceEvent("IncludeServersError").error(e, true);
+			TraceEvent("IncludeServersError").errorUnsuppressed(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index c94f912845..fe93f236c5 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1014,9 +1014,9 @@ struct CLIOptions {
 				} else {
 					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
+					    .error(e)
 					    .detail("Knob", printable(knobName))
-					    .detail("Value", printable(knobValueString))
-					    .error(e);
+					    .detail("Value", printable(knobValueString));
 					exit_code = FDB_EXIT_ERROR;
 				}
 			}
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index a5219ab524..bfef037be5 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -305,9 +305,9 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			throw;
 
 		TraceEvent m(SevWarn, "BackupContainer");
+		m.error(e);
 		m.detail("Description", "Invalid container specification.  See help.");
 		m.detail("URL", url);
-		m.error(e);
 		if (e.code() == error_code_backup_invalid_url)
 			m.detail("LastOpenError", lastOpenError);
 
@@ -360,10 +360,9 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
 			throw;
 
 		TraceEvent m(SevWarn, "BackupContainer");
-
+		m.error(e);
 		m.detail("Description", "Invalid backup container URL prefix.  See help.");
 		m.detail("URL", baseURL);
-		m.error(e);
 		if (e.code() == error_code_backup_invalid_url)
 			m.detail("LastOpenError", IBackupContainer::lastOpenError);
 
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp
index 676ae8165e..22c5087bc0 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@@ -1149,8 +1149,8 @@ public:
 			keyFile = _keyFile;
 		} catch (Error& e) {
 			TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile")
-			    .detail("FileName", encryptionKeyFileName)
-			    .error(e);
+			    .error(e)
+			    .detail("FileName", encryptionKeyFileName);
 			throw e;
 		}
 		int bytesRead = wait(keyFile->read(cipherKey->data(), cipherKey->size(), 0));
@@ -1377,8 +1377,8 @@ ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupConta
 			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
 				// blob http request failure, retry
 				TraceEvent(SevWarnAlways, "BackupContainerGetSnapshotFileKeyRangeConnectionFailure")
-				    .detail("Retries", ++readFileRetries)
-				    .error(e);
+				    .error(e)
+				    .detail("Retries", ++readFileRetries);
 				wait(delayJittered(0.1));
 			} else {
 				TraceEvent(SevError, "BackupContainerGetSnapshotFileKeyRangeUnexpectedError").error(e);
@@ -1549,9 +1549,9 @@ Reference<BackupContainerFileSystem> BackupContainerFileSystem::openContainerFS(
 			throw;
 
 		TraceEvent m(SevWarn, "BackupContainer");
+		m.error(e);
 		m.detail("Description", "Invalid container specification.  See help.");
 		m.detail("URL", url);
-		m.error(e);
 		if (e.code() == error_code_backup_invalid_url)
 			m.detail("LastOpenError", lastOpenError);
 
diff --git a/fdbclient/ConfigKnobs.cpp b/fdbclient/ConfigKnobs.cpp
index 208fb8ff62..f2a1338bf8 100644
--- a/fdbclient/ConfigKnobs.cpp
+++ b/fdbclient/ConfigKnobs.cpp
@@ -27,7 +27,7 @@ ConfigKey ConfigKeyRef::decodeKey(KeyRef const& key) {
 	try {
 		tuple = Tuple::unpack(key);
 	} catch (Error& e) {
-		TraceEvent(SevWarnAlways, "FailedToUnpackConfigKey").detail("Key", printable(key)).error(e);
+		TraceEvent(SevWarnAlways, "FailedToUnpackConfigKey").error(e).detail("Key", printable(key));
 		throw invalid_config_db_key();
 	}
 	if (tuple.size() != 2) {
diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp
index fe072e29fa..7e41b58eb6 100644
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.actor.cpp
@@ -2142,7 +2142,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 				wait(tr->commit());
 				break;
 			} catch (Error& e) {
-				TraceEvent("SetDestUidOrBeginVersionError").error(e, true);
+				TraceEvent("SetDestUidOrBeginVersionError").errorUnsuppressed(e);
 				wait(tr->onError(e));
 			}
 		}
@@ -2907,7 +2907,7 @@ public:
 				TraceEvent("DBA_Abort").detail("CommitVersion", tr->getCommittedVersion());
 				break;
 			} catch (Error& e) {
-				TraceEvent("DBA_AbortError").error(e, true);
+				TraceEvent("DBA_AbortError").errorUnsuppressed(e);
 				wait(tr->onError(e));
 			}
 		}
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 7546226673..d431c763c5 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4407,9 +4407,9 @@ public:
 				break;
 			} catch (Error& e) {
 				TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsMayFail")
+				    .error(e)
 				    .detail("Reason", "DB is not properly locked")
-				    .detail("ExpectedLockID", randomUID)
-				    .error(e);
+				    .detail("ExpectedLockID", randomUID);
 				numTries++;
 				wait(tr->onError(e));
 			}
@@ -4443,8 +4443,8 @@ public:
 				break;
 			} catch (Error& e) {
 				TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsRetry")
-				    .detail("RestoreIndex", restoreIndex)
-				    .error(e);
+				    .error(e)
+				    .detail("RestoreIndex", restoreIndex);
 				numTries++;
 				wait(tr->onError(e));
 			}
@@ -5800,9 +5800,9 @@ ACTOR static Future<Void> transformDatabaseContents(Database cx,
 			break;
 		} catch (Error& e) {
 			TraceEvent("FastRestoreWorkloadTransformDatabaseContentsGetAllKeys")
+			    .error(e)
 			    .detail("Index", i)
-			    .detail("RestoreRange", restoreRanges[i])
-			    .error(e);
+			    .detail("RestoreRange", restoreRanges[i]);
 			oldData = Standalone<VectorRef<KeyValueRef>>(); // clear the vector
 			wait(tr.onError(e));
 		}
diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp
index b34504b641..d10445d38d 100644
--- a/fdbclient/HTTP.actor.cpp
+++ b/fdbclient/HTTP.actor.cpp
@@ -448,6 +448,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 				err = http_bad_request_id();
 
 				TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
+				    .error(err.get())
 				    .detail("DebugID", conn->getDebugID())
 				    .detail("RemoteAddress", conn->getPeerAddress())
 				    .detail("Verb", verb)
@@ -456,8 +457,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 				    .detail("ResponseCode", r->code)
 				    .detail("ResponseContentLen", r->contentLen)
 				    .detail("RequestIDSent", requestID)
-				    .detail("RequestIDReceived", responseID)
-				    .error(err.get());
+				    .detail("RequestIDReceived", responseID);
 			}
 		}
 
@@ -501,7 +501,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 			       contentLen,
 			       total_sent);
 		}
-		event.error(e);
+		event.errorUnsuppressed(e);
 		throw;
 	}
 }
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index cf5763b8b5..0239cd23af 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1273,7 +1273,7 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 				wait(ryw.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("ExcludeServersError").error(e, true);
+				TraceEvent("ExcludeServersError").errorUnsuppressed(e);
 				wait(ryw.onError(e));
 			}
 		}
@@ -1285,7 +1285,7 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("ExcludeServersError").error(e, true);
+				TraceEvent("ExcludeServersError").errorUnsuppressed(e);
 				wait(tr.onError(e));
 			}
 		}
@@ -1336,7 +1336,7 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 				wait(ryw.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("ExcludeLocalitiesError").error(e, true);
+				TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(e);
 				wait(ryw.onError(e));
 			}
 		}
@@ -1348,7 +1348,7 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("ExcludeLocalitiesError").error(e, true);
+				TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(e);
 				wait(tr.onError(e));
 			}
 		}
@@ -1392,7 +1392,7 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 				wait(ryw.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("IncludeServersError").error(e, true);
+				TraceEvent("IncludeServersError").errorUnsuppressed(e);
 				wait(ryw.onError(e));
 			}
 		}
@@ -1449,7 +1449,7 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("IncludeServersError").error(e, true);
+				TraceEvent("IncludeServersError").errorUnsuppressed(e);
 				wait(tr.onError(e));
 			}
 		}
@@ -1487,7 +1487,7 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 				wait(ryw.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("IncludeLocalitiesError").error(e, true);
+				TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
 				wait(ryw.onError(e));
 			}
 		}
@@ -1535,7 +1535,7 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
-				TraceEvent("IncludeLocalitiesError").error(e, true);
+				TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
 				wait(tr.onError(e));
 			}
 		}
@@ -1907,7 +1907,7 @@ ACTOR Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UI
 		TraceEvent("SnapCreateSucceeded").detail("snapUID", snapUID);
 		return Void();
 	} catch (Error& e) {
-		TraceEvent(SevWarn, "SnapCreateFailed").detail("snapUID", snapUID).error(e);
+		TraceEvent(SevWarn, "SnapCreateFailed").error(e).detail("snapUID", snapUID);
 		throw;
 	}
 }
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index abfda0aefd..4ba785cd42 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -562,9 +562,9 @@ ACTOR Future<Void> monitorNominee(Key key,
 			if (rep.isError()) {
 				// Connecting to nominee failed, most likely due to connection failed.
 				TraceEvent("MonitorNomineeError")
+				    .error(rep.getError())
 				    .detail("Hostname", hostname.present() ? hostname.get().toString() : "UnknownHostname")
-				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString())
-				    .error(rep.getError());
+				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString());
 				if (rep.getError().code() == error_code_request_maybe_delivered) {
 					// 50 milliseconds delay to prevent tight resolving loop due to outdated DNS cache
 					wait(delay(0.05));
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index eabe98dd74..39dac56bf9 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -1202,9 +1202,9 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
 					// but we may not see trace logs from this client until a successful connection
 					// is established.
 					TraceEvent(SevWarnAlways, "FailedToInitializeExternalClient")
+					    .error(e)
 					    .detail("LibraryPath", client->libPath)
-					    .detail("ClusterFilePath", clusterFilePath)
-					    .error(e);
+					    .detail("ClusterFilePath", clusterFilePath);
 				}
 			}
 		});
@@ -1218,9 +1218,9 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
 				} catch (Error& e) {
 					// This connection is discarded
 					TraceEvent(SevWarnAlways, "FailedToCreateLegacyDatabaseConnection")
+					    .error(e)
 					    .detail("LibraryPath", client->libPath)
-					    .detail("ClusterFilePath", clusterFilePath)
-					    .error(e);
+					    .detail("ClusterFilePath", clusterFilePath);
 				}
 			}
 		});
@@ -1360,8 +1360,8 @@ ThreadFuture<Void> MultiVersionDatabase::DatabaseState::monitorProtocolVersion()
 			}
 
 			TraceEvent("ErrorGettingClusterProtocolVersion")
-			    .detail("ExpectedProtocolVersion", expected)
-			    .error(cv.getError());
+			    .error(cv.getError())
+			    .detail("ExpectedProtocolVersion", expected);
 		}
 
 		ProtocolVersion clusterVersion =
@@ -1409,10 +1409,10 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
 				newDb = client->api->createDatabase(clusterFilePath.c_str());
 			} catch (Error& e) {
 				TraceEvent(SevWarnAlways, "MultiVersionClientFailedToCreateDatabase")
+				    .error(e)
 				    .detail("LibraryPath", client->libPath)
 				    .detail("External", client->external)
-				    .detail("ClusterFilePath", clusterFilePath)
-				    .error(e);
+				    .detail("ClusterFilePath", clusterFilePath);
 
 				// Put the client in a disconnected state until the version changes again
 				updateDatabase(Reference<IDatabase>(), Reference<ClientInfo>());
@@ -1486,8 +1486,8 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 				// We can't create a new database to monitor the cluster version. This means we will continue using the
 				// previous one, which should hopefully continue to work.
 				TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring")
-				    .detail("ClusterFilePath", clusterFilePath)
-				    .error(e);
+				    .error(e)
+				    .detail("ClusterFilePath", clusterFilePath);
 			}
 		}
 	} else {
@@ -1499,8 +1499,8 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 			// We can't create a new database to monitor the cluster version. This means we will continue using the
 			// previous one, which should hopefully continue to work.
 			TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring")
-			    .detail("ClusterFilePath", clusterFilePath)
-			    .error(e);
+			    .error(e)
+			    .detail("ClusterFilePath", clusterFilePath);
 		}
 	}
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index ace47c6c97..09d55c2c7f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -5097,7 +5097,7 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
 			return Void();
 		} catch (Error& e) {
 			TraceEvent("CommitDummyTransactionError")
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("Key", range.begin)
 			    .detail("Retries", retries);
 			wait(tr.onError(e));
@@ -6849,7 +6849,7 @@ ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID sn
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent("SnapCreateError").detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID).error(e);
+		TraceEvent("SnapCreateError").error(e).detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID);
 		throw;
 	}
 }
@@ -6877,9 +6877,9 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled) {
 			TraceEvent("ExclusionSafetyCheckError")
+			    .error(e)
 			    .detail("NumExclusion", exclusions.size())
-			    .detail("Exclusions", describe(exclusions))
-			    .error(e);
+			    .detail("Exclusions", describe(exclusions));
 		}
 		throw;
 	}
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index fdc1d5c771..ee021b6a05 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2585,7 +2585,7 @@ void ReadYourWritesTransaction::debugLogRetries(Optional<Error> error) {
 			{
 				TraceEvent trace = TraceEvent("LongTransaction");
 				if (error.present())
-					trace.error(error.get(), true);
+					trace.errorUnsuppressed(error.get());
 				if (!transactionDebugInfo->transactionName.empty())
 					trace.detail("TransactionName", transactionDebugInfo->transactionName);
 				trace.detail("Elapsed", elapsed).detail("Retries", retries).detail("Committed", committed);
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index aba8d1667e..0abcbe441a 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -500,7 +500,7 @@ ACTOR Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path) {
 
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled)
-			TraceEvent(SevWarn, errorEventType).error(e).suppressFor(60).detail("File", path);
+			TraceEvent(SevWarn, errorEventType).errorUnsuppressed(e).suppressFor(60).detail("File", path);
 	}
 
 	return Optional<json_spirit::mObject>();
@@ -744,7 +744,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 
 		// Attach err to trace event if present, otherwise extract some stuff from the response
 		if (err.present()) {
-			event.error(err.get());
+			event.errorUnsuppressed(err.get());
 		}
 		event.suppressFor(60);
 		if (!err.present()) {
@@ -954,7 +954,7 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled)
 				TraceEvent(SevWarn, "S3BlobStoreEndpointListResultParseError")
-				    .error(e)
+				    .errorUnsuppressed(e)
 				    .suppressFor(60)
 				    .detail("Resource", fullResource);
 			throw http_bad_response();
@@ -1080,7 +1080,7 @@ ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<S3BlobStoreEnd
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled)
 				TraceEvent(SevWarn, "S3BlobStoreEndpointListBucketResultParseError")
-				    .error(e)
+				    .errorUnsuppressed(e)
 				    .suppressFor(60)
 				    .detail("Resource", fullResource);
 			throw http_bad_response();
diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h
index ec659d68fb..ffbe7746a6 100644
--- a/fdbrpc/AsyncFileNonDurable.actor.h
+++ b/fdbrpc/AsyncFileNonDurable.actor.h
@@ -260,7 +260,7 @@ public:
 			std::string currentFilename =
 			    (wrappedFile.isReady() && !wrappedFile.isError()) ? wrappedFile.get()->getFilename() : actualFilename;
 			currentProcess->machine->openFiles.erase(currentFilename);
-			//TraceEvent("AsyncFileNonDurableOpenError").error(e, true).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator.getCurrentProcess()->address);
+			//TraceEvent("AsyncFileNonDurableOpenError").errorUnsuppressed(e).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator.getCurrentProcess()->address);
 			wait(g_simulator.onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 952ca20367..3e3b13709a 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -732,13 +732,13 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 
 			if (self->compatible) {
 				TraceEvent(ok ? SevInfo : SevWarnAlways, "ConnectionClosed", conn ? conn->getDebugID() : UID())
-				    .error(e, true)
+				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
 				    .detail("PeerAddr", self->destination);
 			} else {
 				TraceEvent(
 				    ok ? SevInfo : SevWarnAlways, "IncompatibleConnectionClosed", conn ? conn->getDebugID() : UID())
-				    .error(e, true)
+				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
 				    .detail("PeerAddr", self->destination);
 			}
@@ -783,7 +783,7 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 
 			if (self->peerReferences <= 0 && self->reliable.empty() && self->unsent.empty() &&
 			    self->outstandingReplies == 0) {
-				TraceEvent("PeerDestroy").error(e).suppressFor(1.0).detail("PeerAddr", self->destination);
+				TraceEvent("PeerDestroy").errorUnsuppressed(e).suppressFor(1.0).detail("PeerAddr", self->destination);
 				self->connect.cancel();
 				self->transport->peers.erase(self->destination);
 				self->transport->orderedAddresses.erase(self->destination);
@@ -1330,10 +1330,12 @@ ACTOR static Future<Void> connectionIncoming(TransportData* self, Reference<ICon
 		}
 		return Void();
 	} catch (Error& e) {
-		TraceEvent("IncomingConnectionError", conn->getDebugID())
-		    .error(e)
-		    .suppressFor(1.0)
-		    .detail("FromAddress", conn->getPeerAddress());
+		if (e.code() != error_code_actor_cancelled) {
+			TraceEvent("IncomingConnectionError", conn->getDebugID())
+			    .errorUnsuppressed(e)
+			    .suppressFor(1.0)
+			    .detail("FromAddress", conn->getPeerAddress());
+		}
 		conn->close();
 		return Void();
 	}
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 2a66db8b63..b83f1d81b0 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -2094,7 +2094,7 @@ public:
 				t.action.send(Void());
 				ASSERT(this->currentProcess == t.machine);
 			} catch (Error& e) {
-				TraceEvent(SevError, "UnhandledSimulationEventError").error(e, true);
+				TraceEvent(SevError, "UnhandledSimulationEventError").errorUnsuppressed(e);
 				killProcess(t.machine, KillInstantly);
 			}
 
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 5e311be449..895497fff7 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -1101,10 +1101,10 @@ ACTOR Future<Void> backupWorker(BackupInterface interf,
 			try {
 				wait(done);
 			} catch (Error& e) {
-				TraceEvent("BackupWorkerShutdownError", self.myId).error(e, true);
+				TraceEvent("BackupWorkerShutdownError", self.myId).errorUnsuppressed(e);
 			}
 		}
-		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);
+		TraceEvent("BackupWorkerTerminated", self.myId).errorUnsuppressed(err);
 		if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
 			throw err;
 		}
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index f03b23e502..f04360b473 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -843,8 +843,8 @@ ACTOR Future<Void> monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerIn
 				}
 				// TODO change back from SevError?
 				TraceEvent(SevError, "BWStatusMonitoringFailed", bmData->id)
-				    .detail("BlobWorkerID", bwInterf.id())
-				    .error(e);
+				    .error(e)
+				    .detail("BlobWorkerID", bwInterf.id());
 				throw e;
 			}
 		}
@@ -877,7 +877,7 @@ ACTOR Future<Void> monitorBlobWorker(BlobManagerData* bmData, BlobWorkerInterfac
 			printf("BM got unexpected error %s monitoring BW %s\n", e.name(), bwInterf.id().toString().c_str());
 		}
 		// TODO change back from SevError?
-		TraceEvent(SevError, "BWMonitoringFailed", bmData->id).detail("BlobWorkerID", bwInterf.id()).error(e);
+		TraceEvent(SevError, "BWMonitoringFailed", bmData->id).error(e).detail("BlobWorkerID", bwInterf.id());
 		throw e;
 	}
 
@@ -1152,7 +1152,7 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 			}
 		}
 	} catch (Error& err) {
-		TraceEvent("BlobManagerDied", bmInterf.id()).error(err, true);
+		TraceEvent("BlobManagerDied", bmInterf.id()).errorUnsuppressed(err);
 	}
 	return Void();
 }
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 2c070e7760..66b7714854 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -1589,7 +1589,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				       metadata->keyRange.end.printable().c_str(),
 				       e.name());
 			}
-			TraceEvent(SevWarn, "GranuleFileUpdaterError", bwData->id).detail("Granule", metadata->keyRange).error(e);
+			TraceEvent(SevWarn, "GranuleFileUpdaterError", bwData->id).error(e).detail("Granule", metadata->keyRange);
 
 			if (granuleCanRetry(e)) {
 				// explicitly cancel all outstanding write futures BEFORE updating promise stream, to ensure they
@@ -2621,7 +2621,7 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 		if (BW_DEBUG) {
 			printf("Blob worker got error %s. Exiting...\n", e.name());
 		}
-		TraceEvent("BlobWorkerDied", self->id).error(e, true);
+		TraceEvent("BlobWorkerDied", self->id).errorUnsuppressed(e);
 	}
 
 	wait(self->granuleMetadata.clearAsync());
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 99eac07eaa..256371852b 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -296,7 +296,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			TraceEvent(SevWarn, "DetectedFailedRecovery", cluster->id).detail("OldMaster", iMaster.id());
 		} catch (Error& e) {
 			state Error err = e;
-			TraceEvent("CCWDB", cluster->id).error(e, true).detail("Master", iMaster.id());
+			TraceEvent("CCWDB", cluster->id).errorUnsuppressed(e).detail("Master", iMaster.id());
 			if (e.code() != error_code_actor_cancelled)
 				wait(delay(0.0));
 
@@ -313,7 +313,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			TEST(err.code() == error_code_restart_cluster_controller); // Terminated due to cluster-controller restart.
 
 			if (cluster->shouldCommitSuicide || err.code() == error_code_coordinators_changed) {
-				TraceEvent("ClusterControllerTerminate", cluster->id).error(err, true);
+				TraceEvent("ClusterControllerTerminate", cluster->id).errorUnsuppressed(err);
 				throw restart_cluster_controller();
 			}
 
@@ -427,10 +427,10 @@ void checkOutstandingStorageRequests(ClusterControllerData* self) {
 		} catch (Error& e) {
 			if (e.code() == error_code_no_more_servers) {
 				TraceEvent(SevWarn, "RecruitStorageNotAvailable", self->id)
+				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
 				    .detail("OutstandingReq", i)
-				    .detail("IsCriticalRecruitment", req.first.criticalRecruitment)
-				    .error(e);
+				    .detail("IsCriticalRecruitment", req.first.criticalRecruitment);
 			} else {
 				TraceEvent(SevError, "RecruitStorageError", self->id).error(e);
 				throw;
@@ -464,9 +464,9 @@ void checkOutstandingBlobWorkerRequests(ClusterControllerData* self) {
 		} catch (Error& e) {
 			if (e.code() == error_code_no_more_servers) {
 				TraceEvent(SevWarn, "RecruitBlobWorkerNotAvailable", self->id)
+				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
-				    .detail("OutstandingReq", i)
-				    .error(e);
+				    .detail("OutstandingReq", i);
 			} else {
 				TraceEvent(SevError, "RecruitBlobWorkerError", self->id).error(e);
 				throw;
@@ -876,8 +876,8 @@ void clusterRecruitStorage(ClusterControllerData* self, RecruitStorageRequest re
 		if (e.code() == error_code_no_more_servers) {
 			self->outstandingStorageRequests.emplace_back(req, now() + SERVER_KNOBS->RECRUITMENT_TIMEOUT);
 			TraceEvent(SevWarn, "RecruitStorageNotAvailable", self->id)
-			    .detail("IsCriticalRecruitment", req.criticalRecruitment)
-			    .error(e);
+			    .error(e)
+			    .detail("IsCriticalRecruitment", req.criticalRecruitment);
 		} else {
 			TraceEvent(SevError, "RecruitStorageError", self->id).error(e);
 			throw; // Any other error will bring down the cluster controller
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 4911d4b6ed..8a1fa1e3cf 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -1896,8 +1896,8 @@ public:
 					throw;
 				}
 				TraceEvent(SevWarn, "AttemptingRecruitmentInRemoteDc", id)
-				    .detail("SetPrimaryDesired", setPrimaryDesired)
-				    .error(e);
+				    .error(e)
+				    .detail("SetPrimaryDesired", setPrimaryDesired);
 				auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId, checkGoodRecruitment);
 				if (!setPrimaryDesired) {
 					std::vector<Optional<Key>> dcPriority;
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index eb826a0aa6..17429f6cea 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -1769,17 +1769,17 @@ ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co
 			wait(throwErrorOr(ddSnapReq));
 		} catch (Error& e) {
 			TraceEvent("SnapCommitProxy_DDSnapResponseError")
+			    .errorUnsuppressed(e)
 			    .detail("SnapPayload", snapReq.snapPayload)
-			    .detail("SnapUID", snapReq.snapUID)
-			    .error(e, true /*includeCancelled*/);
+			    .detail("SnapUID", snapReq.snapUID);
 			throw e;
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
 		TraceEvent("SnapCommitProxy_SnapReqError")
+		    .errorUnsuppressed(e)
 		    .detail("SnapPayload", snapReq.snapPayload)
-		    .detail("SnapUID", snapReq.snapUID)
-		    .error(e, true /*includeCancelled*/);
+		    .detail("SnapUID", snapReq.snapUID);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
@@ -2188,7 +2188,7 @@ ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
 		                                                whitelistBinPaths);
 		wait(core || checkRemoved(db, req.recoveryCount, proxy));
 	} catch (Error& e) {
-		TraceEvent("CommitProxyTerminated", proxy.id()).error(e, true);
+		TraceEvent("CommitProxyTerminated", proxy.id()).errorUnsuppressed(e);
 
 		if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped &&
 		    e.code() != error_code_tlog_failed && e.code() != error_code_coordinators_changed &&
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index d334de8bc5..b2814c106b 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -796,7 +796,7 @@ ACTOR Future<Void> coordinationServer(std::string dataFolder,
 		     store.getError() || configDatabaseServer);
 		throw internal_error();
 	} catch (Error& e) {
-		TraceEvent("CoordinationServerError", myID).error(e, true);
+		TraceEvent("CoordinationServerError", myID).errorUnsuppressed(e);
 		throw;
 	}
 }
diff --git a/fdbserver/CoroFlow.actor.cpp b/fdbserver/CoroFlow.actor.cpp
index bf37ec7053..948e1f8fd1 100644
--- a/fdbserver/CoroFlow.actor.cpp
+++ b/fdbserver/CoroFlow.actor.cpp
@@ -178,7 +178,7 @@ class WorkPool final : public IThreadPool, public ReferenceCounted<WorkPool<Thre
 				stopped.send(Void());
 				return;
 			} catch (Error& e) {
-				TraceEvent("WorkPoolError").error(e, true);
+				TraceEvent("WorkPoolError").errorUnsuppressed(e);
 				error.sendError(e);
 			} catch (...) {
 				TraceEvent("WorkPoolError").log();
@@ -259,7 +259,7 @@ public:
 		    .detail("Workers", pool->workers.size())
 		    .detail("Idle", pool->idle.size())
 		    .detail("Work", pool->work.size())
-		    .error(e, true);
+		    .errorUnsuppressed(e);
 
 		for (uint32_t i = 0; i < pool->work.size(); i++)
 			pool->work[i]->cancel(); // What if cancel() does something to this?
diff --git a/fdbserver/CoroFlowCoro.actor.cpp b/fdbserver/CoroFlowCoro.actor.cpp
index 9ff42a1322..bf9c626725 100644
--- a/fdbserver/CoroFlowCoro.actor.cpp
+++ b/fdbserver/CoroFlowCoro.actor.cpp
@@ -154,7 +154,7 @@ class WorkPool final : public IThreadPool, public ReferenceCounted<WorkPool<Thre
 				stopped.send(Void());
 				return;
 			} catch (Error& e) {
-				TraceEvent("WorkPoolError").error(e, true);
+				TraceEvent("WorkPoolError").errorUnsuppressed(e);
 				error.sendError(e);
 			} catch (...) {
 				TraceEvent("WorkPoolError").log();
@@ -232,10 +232,10 @@ public:
 
 		pool->queueLock.enter();
 		TraceEvent("WorkPool_Stop")
+		    .errorUnsuppressed(e)
 		    .detail("Workers", pool->workers.size())
 		    .detail("Idle", pool->idle.size())
-		    .detail("Work", pool->work.size())
-		    .error(e, true);
+		    .detail("Work", pool->work.size());
 
 		for (uint32_t i = 0; i < pool->work.size(); i++)
 			pool->work[i]->cancel(); // What if cancel() does something to this?
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index fa564ea040..050d91350b 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1331,10 +1331,10 @@ public:
 		} catch (Error& e) {
 			state Error err = e;
 			TraceEvent("StorageServerTrackerCancelled", self->distributorId)
+			    .errorUnsuppressed(e)
 			    .suppressFor(1.0)
 			    .detail("Primary", self->primary)
-			    .detail("Server", server->getId())
-			    .error(e, /*includeCancelled*/ true);
+			    .detail("Server", server->getId());
 			if (e.code() != error_code_actor_cancelled && errorOut.canBeSet()) {
 				errorOut.sendError(e);
 				wait(delay(0)); // Check for cancellation, since errorOut.sendError(e) could delete self
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 31e059d594..a012445d52 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -865,7 +865,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 				}
 
 				bool ddEnabled = wait(isDataDistributionEnabled(cx, ddEnabledState));
-				TraceEvent("DataDistributionMoveKeysConflict").detail("DataDistributionEnabled", ddEnabled).error(err);
+				TraceEvent("DataDistributionMoveKeysConflict").error(err).detail("DataDistributionEnabled", ddEnabled);
 				if (ddEnabled) {
 					throw err;
 				}
@@ -891,7 +891,7 @@ Future<Void> sendSnapReq(RequestStream<Req> stream, Req req, Error e) {
 	ErrorOr<REPLY_TYPE(Req)> reply = wait(stream.tryGetReply(req));
 	if (reply.isError()) {
 		TraceEvent("SnapDataDistributor_ReqError")
-		    .error(reply.getError(), true)
+		    .errorUnsuppressed(reply.getError())
 		    .detail("ConvertedErrorType", e.what())
 		    .detail("Peer", stream.getEndpoint().getPrimaryAddress());
 		throw e;
@@ -1012,9 +1012,9 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 	} catch (Error& err) {
 		state Error e = err;
 		TraceEvent("SnapDataDistributor_SnapReqExit")
+		    .errorUnsuppressed(e)
 		    .detail("SnapPayload", snapReq.snapPayload)
-		    .detail("SnapUID", snapReq.snapUID)
-		    .error(e, true /*includeCancelled */);
+		    .detail("SnapUID", snapReq.snapUID);
 		if (e.code() == error_code_snap_storage_failed || e.code() == error_code_snap_tlog_failed ||
 		    e.code() == error_code_operation_cancelled || e.code() == error_code_snap_disable_tlog_pop_failed) {
 			// enable tlog pop on local tlog nodes
@@ -1072,9 +1072,9 @@ ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq,
 		}
 	} catch (Error& e) {
 		TraceEvent("SnapDDCreateError")
+		    .errorUnsuppressed(e)
 		    .detail("SnapPayload", snapReq.snapPayload)
-		    .detail("SnapUID", snapReq.snapUID)
-		    .error(e, true /*includeCancelled */);
+		    .detail("SnapUID", snapReq.snapUID);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
@@ -1251,10 +1251,10 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 		}
 	} catch (Error& err) {
 		if (normalDataDistributorErrors().count(err.code()) == 0) {
-			TraceEvent("DataDistributorError", di.id()).error(err, true);
+			TraceEvent("DataDistributorError", di.id()).errorUnsuppressed(err);
 			throw err;
 		}
-		TraceEvent("DataDistributorDied", di.id()).error(err, true);
+		TraceEvent("DataDistributorDied", di.id()).errorUnsuppressed(err);
 	}
 
 	return Void();
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index fcbc73dbed..55d86742fa 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -1261,10 +1261,12 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent(relocateShardInterval.end(), distributorId).error(e, true).detail("Duration", now() - startTime);
+		TraceEvent(relocateShardInterval.end(), distributorId)
+		    .errorUnsuppressed(e)
+		    .detail("Duration", now() - startTime);
 		if (now() - startTime > 600) {
 			TraceEvent(SevWarnAlways, "RelocateShardTooLong")
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("Duration", now() - startTime)
 			    .detail("Dest", describe(destIds))
 			    .detail("Src", describe(rd.src));
@@ -1448,8 +1450,8 @@ ACTOR Future<Void> BgDDMountainChopper(DDQueueData* self, int teamCollectionInde
 			traceEvent.detail("ResetCount", resetCount);
 			tr.reset();
 		} catch (Error& e) {
-			traceEvent.error(
-			    e, true); // Log actor_cancelled because it's not legal to suppress an event that's initialized
+			// Log actor_cancelled because it's not legal to suppress an event that's initialized
+			traceEvent.errorUnsuppressed(e);
 			wait(tr.onError(e));
 		}
 
@@ -1554,8 +1556,8 @@ ACTOR Future<Void> BgDDValleyFiller(DDQueueData* self, int teamCollectionIndex)
 			traceEvent.detail("ResetCount", resetCount);
 			tr.reset();
 		} catch (Error& e) {
-			traceEvent.error(
-			    e, true); // Log actor_cancelled because it's not legal to suppress an event that's initialized
+			// Log actor_cancelled because it's not legal to suppress an event that's initialized
+			traceEvent.errorUnsuppressed(e);
 			wait(tr.onError(e));
 		}
 
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 52f737e5f1..a955bafa4f 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -492,7 +492,9 @@ public:
 			delete pageMem;
 			TEST(true); // push error
 			TEST(2 == syncFiles.size()); // push spanning both files error
-			TraceEvent(SevError, "RDQPushAndCommitError", dbgid).error(e, true).detail("InitialFilename0", filename);
+			TraceEvent(SevError, "RDQPushAndCommitError", dbgid)
+			    .errorUnsuppressed(e)
+			    .detail("InitialFilename0", filename);
 
 			if (errorPromise.canBeSet())
 				errorPromise.sendError(e);
@@ -612,7 +614,7 @@ public:
 			    .detail("File0", self->filename(0));
 		} catch (Error& e) {
 			TraceEvent(SevError, "DiskQueueShutdownError", self->dbgid)
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("Reason", e.code() == error_code_platform_error ? "could not delete database" : "unknown");
 			error = e;
 		}
@@ -731,7 +733,7 @@ public:
 		} catch (Error& e) {
 			bool ok = e.code() == error_code_file_not_found;
 			TraceEvent(ok ? SevInfo : SevError, "RDQReadFirstAndLastPagesError", self->dbgid)
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("File0Name", self->files[0].dbgFilename);
 			if (!self->error.isSet())
 				self->error.sendError(e);
@@ -804,7 +806,7 @@ public:
 		} catch (Error& e) {
 			TEST(true); // Read next page error
 			TraceEvent(SevError, "RDQReadNextPageError", self->dbgid)
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("File0Name", self->files[0].dbgFilename);
 			if (!self->error.isSet())
 				self->error.sendError(e);
diff --git a/fdbserver/EncryptKeyProxy.actor.cpp b/fdbserver/EncryptKeyProxy.actor.cpp
index be602f0865..0f666b0f53 100644
--- a/fdbserver/EncryptKeyProxy.actor.cpp
+++ b/fdbserver/EncryptKeyProxy.actor.cpp
@@ -58,7 +58,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent("EKP_Terminated", ekpInterface.id()).error(e, true);
+		TraceEvent("EKP_Terminated", ekpInterface.id()).errorUnsuppressed(e);
 	}
 
 	return Void();
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 5f427dd0b0..c903e85d5d 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -991,7 +991,7 @@ ACTOR Future<Void> grvProxyServer(GrvProxyInterface proxy,
 		state Future<Void> core = grvProxyServerCore(proxy, req.master, req.masterLifetime, db);
 		wait(core || checkRemoved(db, req.recoveryCount, proxy));
 	} catch (Error& e) {
-		TraceEvent("GrvProxyTerminated", proxy.id()).error(e, true);
+		TraceEvent("GrvProxyTerminated", proxy.id()).errorUnsuppressed(e);
 
 		if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped &&
 		    e.code() != error_code_tlog_failed && e.code() != error_code_coordinators_changed &&
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp
index 0b7833956f..563e7db736 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/KeyValueStoreMemory.actor.cpp
@@ -634,7 +634,7 @@ private:
 			} catch (Error& e) {
 				bool ok = e.code() == error_code_operation_cancelled || e.code() == error_code_file_not_found ||
 				          e.code() == error_code_disk_adapter_reset;
-				TraceEvent(ok ? SevInfo : SevError, "ErrorDuringRecovery", dbgid).error(e, true);
+				TraceEvent(ok ? SevInfo : SevError, "ErrorDuringRecovery", dbgid).errorUnsuppressed(e);
 				if (e.code() != error_code_disk_adapter_reset) {
 					throw e;
 				}
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 5384e5cc5b..571c7d49c9 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -2061,8 +2061,8 @@ private:
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "KVDoCloseError", self->logID)
+			    .errorUnsuppressed(e)
 			    .detail("Filename", self->filename)
-			    .error(e, true)
 			    .detail("Reason", e.code() == error_code_platform_error ? "could not delete database" : "unknown");
 			error = e;
 		}
diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index bb07cdbb52..43abdac623 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -627,8 +627,8 @@ ACTOR Future<Void> logRouterPeekStream(LogRouterData* self, TLogPeekStreamReques
 		} catch (Error& e) {
 			self->activePeekStreams--;
 			TraceEvent(SevDebug, "TLogPeekStreamEnd", self->dbgid)
-			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress())
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
@@ -764,7 +764,7 @@ ACTOR Future<Void> logRouter(TLogInterface interf,
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled || e.code() == error_code_worker_removed) {
-			TraceEvent("LogRouterTerminated", interf.id()).error(e, true);
+			TraceEvent("LogRouterTerminated", interf.id()).errorUnsuppressed(e);
 			return Void();
 		}
 		throw;
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index f3a07e8062..8ee250093a 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -367,7 +367,7 @@ ACTOR Future<Void> serverPeekStreamGetMore(ILogSystem::ServerPeekCursor* self, T
 				}
 			}
 		} catch (Error& e) {
-			DisabledTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID).error(e, true);
+			DisabledTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID).errorUnsuppressed(e);
 			if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete) {
 				// NOTE: delay in order to avoid the endless retry loop block other tasks
 				self->peekReplyStream.reset();
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp
index 9109f0547b..115d65be7a 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/MoveKeys.actor.cpp
@@ -558,7 +558,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 		    .detail("Shards", shards)
 		    .detail("MaxRetries", maxRetries);
 	} catch (Error& e) {
-		TraceEvent(SevDebug, interval.end(), relocationIntervalId).error(e, true);
+		TraceEvent(SevDebug, interval.end(), relocationIntervalId).errorUnsuppressed(e);
 		throw;
 	}
 
@@ -992,7 +992,7 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 
 		TraceEvent(SevDebug, interval.end(), relocationIntervalId);
 	} catch (Error& e) {
-		TraceEvent(SevDebug, interval.end(), relocationIntervalId).error(e, true);
+		TraceEvent(SevDebug, interval.end(), relocationIntervalId).errorUnsuppressed(e);
 		throw;
 	}
 	return Void();
diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp
index 0acae5cdac..eec751cbea 100644
--- a/fdbserver/MutationTracking.cpp
+++ b/fdbserver/MutationTracking.cpp
@@ -41,7 +41,7 @@ TraceEvent debugMutationEnabled(const char* context, Version version, MutationRe
 
 	for (auto& labelKey : debugKeys) {
 		if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
-		     KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
+		    KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
 		    mutation.param1 == labelKey.second) {
 			label = labelKey.first;
 			break;
diff --git a/fdbserver/OldTLogServer_4_6.actor.cpp b/fdbserver/OldTLogServer_4_6.actor.cpp
index eca9c17390..47b79cc740 100644
--- a/fdbserver/OldTLogServer_4_6.actor.cpp
+++ b/fdbserver/OldTLogServer_4_6.actor.cpp
@@ -1161,8 +1161,8 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 		} catch (Error& e) {
 			self->activePeekStreams--;
 			TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId)
-			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress())
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
@@ -1646,7 +1646,7 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 		wait(error);
 		throw internal_error();
 	} catch (Error& e) {
-		TraceEvent("TLogError", tlogId).error(e, true);
+		TraceEvent("TLogError", tlogId).errorUnsuppressed(e);
 
 		for (auto& it : self.id_data) {
 			if (it.second->recoverySuccessful.canBeSet()) {
diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp
index 98e8d74672..23d2a67d48 100644
--- a/fdbserver/OldTLogServer_6_0.actor.cpp
+++ b/fdbserver/OldTLogServer_6_0.actor.cpp
@@ -1479,8 +1479,8 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 		} catch (Error& e) {
 			self->activePeekStreams--;
 			TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId)
-			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress())
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
@@ -1912,7 +1912,7 @@ ACTOR Future<Void> tLogSnapCreate(TLogSnapRequest snapReq, TLogData* self, Refer
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
-		TraceEvent("TLogSnapCreateError").error(e, true /*includeCancelled */);
+		TraceEvent("TLogSnapCreateError").errorUnsuppressed(e);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
@@ -2555,7 +2555,7 @@ bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* p
 
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
 	    e.code() == error_code_file_not_found) {
-		TraceEvent("TLogTerminated", self->dbgid).error(e, true);
+		TraceEvent("TLogTerminated", self->dbgid).errorUnsuppressed(e);
 		return true;
 	} else
 		return false;
@@ -2848,7 +2848,7 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 		}
 	} catch (Error& e) {
 		self.terminated.send(Void());
-		TraceEvent("TLogError", tlogId).error(e, true);
+		TraceEvent("TLogError", tlogId).errorUnsuppressed(e);
 		if (recovered.canBeSet())
 			recovered.send(Void());
 
diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp
index 53be793c7c..550defee1a 100644
--- a/fdbserver/OldTLogServer_6_2.actor.cpp
+++ b/fdbserver/OldTLogServer_6_2.actor.cpp
@@ -1908,8 +1908,8 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 		} catch (Error& e) {
 			self->activePeekStreams--;
 			TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId)
-			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress())
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
@@ -2357,7 +2357,7 @@ ACTOR Future<Void> tLogSnapCreate(TLogSnapRequest snapReq, TLogData* self, Refer
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
-		TraceEvent("TLogExecHelperError").error(e, true /*includeCancelled */);
+		TraceEvent("TLogExecHelperError").errorUnsuppressed(e);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
@@ -3038,7 +3038,7 @@ bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* p
 
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
 	    e.code() == error_code_file_not_found) {
-		TraceEvent("TLogTerminated", self->dbgid).error(e, true);
+		TraceEvent("TLogTerminated", self->dbgid).errorUnsuppressed(e);
 		return true;
 	} else
 		return false;
@@ -3336,7 +3336,7 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 		}
 	} catch (Error& e) {
 		self.terminated.send(Void());
-		TraceEvent("TLogError", tlogId).error(e, true);
+		TraceEvent("TLogError", tlogId).errorUnsuppressed(e);
 		if (recovered.canBeSet())
 			recovered.send(Void());
 
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index 56384d15bd..00a29a1f16 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -732,7 +732,7 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 				}
 			}
 		} catch (Error& e) {
-			TraceEvent(("QuietDatabase" + phase + "Error").c_str()).error(e, true);
+			TraceEvent(("QuietDatabase" + phase + "Error").c_str()).errorUnsuppressed(e);
 			if (e.code() != error_code_actor_cancelled && e.code() != error_code_attribute_not_found &&
 			    e.code() != error_code_timed_out)
 				TraceEvent(("QuietDatabase" + phase + "Error").c_str()).error(e);
diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 84fdf39877..8edb87a0fb 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -138,7 +138,9 @@ public:
 				tr = Transaction(self->db);
 				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
 			} catch (Error& e) {
-				TraceEvent("RatekeeperGetSSListError", self->id).error(e).suppressFor(1.0);
+				if (e.code() != error_code_actor_cancelled) {
+					TraceEvent("RatekeeperGetSSListError", self->id).errorUnsuppressed(e).suppressFor(1.0);
+				}
 				wait(tr.onError(e));
 			}
 		}
@@ -405,7 +407,7 @@ public:
 				}
 			}
 		} catch (Error& err) {
-			TraceEvent("RatekeeperDied", rkInterf.id()).error(err, true);
+			TraceEvent("RatekeeperDied", rkInterf.id()).errorUnsuppressed(err);
 		}
 		return Void();
 	}
diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp
index d5e0c27670..44576d3465 100644
--- a/fdbserver/Resolver.actor.cpp
+++ b/fdbserver/Resolver.actor.cpp
@@ -373,7 +373,7 @@ ACTOR Future<Void> resolver(ResolverInterface resolver,
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled || e.code() == error_code_worker_removed) {
-			TraceEvent("ResolverTerminated", resolver.id()).error(e, true);
+			TraceEvent("ResolverTerminated", resolver.id()).errorUnsuppressed(e);
 			return Void();
 		}
 		throw;
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 072aa53a5f..4dc3ea4b09 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -98,8 +98,8 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
 		} catch (Error& e) {
 			bool isError = e.code() != error_code_operation_cancelled;
 			TraceEvent(isError ? SevError : SevWarnAlways, "FastRestoreApplierError", self->id())
-			    .detail("RequestType", requestTypeStr)
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("RequestType", requestTypeStr);
 			actors.clear(false);
 			break;
 		}
@@ -251,9 +251,9 @@ ACTOR static Future<Void> applyClearRangeMutations(Standalone<VectorRef<KeyRange
 			retries++;
 			if (retries > SERVER_KNOBS->FASTRESTORE_TXN_RETRY_MAX) {
 				TraceEvent(SevWarnAlways, "RestoreApplierApplyClearRangeMutationsStuck", applierID)
+				    .error(e)
 				    .detail("BatchIndex", batchIndex)
-				    .detail("ClearRanges", ranges.size())
-				    .error(e);
+				    .detail("ClearRanges", ranges.size());
 			}
 			wait(tr->onError(e));
 		}
@@ -314,11 +314,13 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 		} catch (Error& e) {
 			cc->fetchTxnRetries += 1;
 			if (retries++ > incompleteStagingKeys.size()) {
-				TraceEvent(SevWarnAlways, "GetAndComputeStagingKeys", applierID)
-				    .suppressFor(1.0)
-				    .detail("RandomUID", randomID)
-				    .detail("BatchIndex", batchIndex)
-				    .error(e);
+				if (e.code() != error_code_actor_cancelled) {
+					TraceEvent(SevWarnAlways, "GetAndComputeStagingKeys", applierID)
+					    .errorUnsuppressed(e)
+					    .suppressFor(1.0)
+					    .detail("RandomUID", randomID)
+					    .detail("BatchIndex", batchIndex);
+				}
 			}
 			wait(tr->onError(e));
 		}
diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp
index a9d0444b74..bbc5e8cd06 100644
--- a/fdbserver/RestoreController.actor.cpp
+++ b/fdbserver/RestoreController.actor.cpp
@@ -136,7 +136,7 @@ ACTOR Future<Void> startRestoreController(Reference<RestoreWorkerData> controlle
 		wait(startProcessRestoreRequests(self, cx) || error);
 	} catch (Error& e) {
 		if (e.code() != error_code_operation_cancelled) {
-			TraceEvent(SevError, "FastRestoreControllerStart").detail("Reason", "Unexpected unhandled error").error(e);
+			TraceEvent(SevError, "FastRestoreControllerStart").error(e).detail("Reason", "Unexpected unhandled error");
 		}
 	}
 
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 9e971c94f8..42135a7825 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -224,7 +224,7 @@ ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled) {
-			TraceEvent(SevError, "FastRestoreLoaderDispatchRequests").error(e, true);
+			TraceEvent(SevError, "FastRestoreLoaderDispatchRequests").errorUnsuppressed(e);
 			throw e;
 		}
 	}
@@ -301,8 +301,8 @@ ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
 		} catch (Error& e) {
 			bool isError = e.code() != error_code_operation_cancelled; // == error_code_broken_promise
 			TraceEvent(isError ? SevError : SevWarnAlways, "FastRestoreLoaderError", self->id())
-			    .detail("RequestType", requestTypeStr)
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("RequestType", requestTypeStr);
 			actors.clear(false);
 			break;
 		}
@@ -513,8 +513,8 @@ ACTOR static Future<Void> parsePartitionedLogFileOnLoader(
 			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
 				// blob http request failure, retry
 				TraceEvent(SevWarnAlways, "FastRestoreDecodedPartitionedLogFileConnectionFailure")
-				    .detail("Retries", ++readFileRetries)
-				    .error(e);
+				    .error(e)
+				    .detail("Retries", ++readFileRetries);
 				wait(delayJittered(0.1));
 			} else {
 				TraceEvent(SevError, "FastRestoreParsePartitionedLogFileOnLoaderUnexpectedError").error(e);
@@ -659,10 +659,10 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 	} catch (Error& e) { // In case ci.samples throws broken_promise due to unstable network
 		if (e.code() == error_code_broken_promise || e.code() == error_code_operation_cancelled) {
 			TraceEvent(SevWarnAlways, "FastRestoreLoaderPhaseLoadFileSendSamples")
-			    .detail("SamplesMessages", samplesMessages)
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("SamplesMessages", samplesMessages);
 		} else {
-			TraceEvent(SevError, "FastRestoreLoaderPhaseLoadFileSendSamplesUnexpectedError").error(e, true);
+			TraceEvent(SevError, "FastRestoreLoaderPhaseLoadFileSendSamplesUnexpectedError").errorUnsuppressed(e);
 		}
 	}
 
@@ -1230,8 +1230,8 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
 				// blob http request failure, retry
 				TraceEvent(SevWarnAlways, "FastRestoreDecodedRangeFileConnectionFailure")
-				    .detail("Retries", ++readFileRetries)
-				    .error(e);
+				    .error(e)
+				    .detail("Retries", ++readFileRetries);
 				wait(delayJittered(0.1));
 			} else {
 				TraceEvent(SevError, "FastRestoreParseRangeFileOnLoaderUnexpectedError").error(e);
@@ -1355,8 +1355,8 @@ ACTOR static Future<Void> parseLogFileToMutationsOnLoader(NotifiedVersion* pProc
 			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
 				// blob http request failure, retry
 				TraceEvent(SevWarnAlways, "FastRestoreDecodedLogFileConnectionFailure")
-				    .detail("Retries", ++readFileRetries)
-				    .error(e);
+				    .error(e)
+				    .detail("Retries", ++readFileRetries);
 				wait(delayJittered(0.1));
 			} else {
 				TraceEvent(SevError, "FastRestoreParseLogFileToMutationsOnLoaderUnexpectedError").error(e);
diff --git a/fdbserver/RestoreWorker.actor.cpp b/fdbserver/RestoreWorker.actor.cpp
index 7bf4010999..d3dc7b2098 100644
--- a/fdbserver/RestoreWorker.actor.cpp
+++ b/fdbserver/RestoreWorker.actor.cpp
@@ -264,7 +264,7 @@ ACTOR Future<Void> startRestoreWorker(Reference<RestoreWorkerData> self, Restore
 				}
 			}
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "FastRestoreWorkerError").detail("RequestType", requestTypeStr).error(e, true);
+			TraceEvent(SevWarn, "FastRestoreWorkerError").errorUnsuppressed(e).detail("RequestType", requestTypeStr);
 			break;
 		}
 	}
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 5fa01027ae..e7cb21e74a 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -601,7 +601,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				               ? SevInfo
 				               : SevError,
 				           "SimulatedFDBDTerminated")
-				    .error(e, true)
+				    .errorUnsuppressed(e)
 				    .detail("ZoneId", localities.zoneId());
 			}
 
@@ -617,7 +617,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				onShutdown = ISimulator::InjectFaults;
 		} catch (Error& e) {
 			TraceEvent(destructed ? SevInfo : SevError, "SimulatedFDBDRebooterError")
-			    .error(e, true)
+			    .errorUnsuppressed(e)
 			    .detail("ZoneId", localities.zoneId())
 			    .detail("RandomId", randomId);
 			onShutdown = e;
diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp
index 12dfb9ec25..ebf3d06d5e 100644
--- a/fdbserver/StorageCache.actor.cpp
+++ b/fdbserver/StorageCache.actor.cpp
@@ -1375,7 +1375,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 				break;
 			} catch (Error& e) {
 				TraceEvent("SCFKBlockFail", data->thisServerID)
-				    .error(e, true)
+				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
 				    .detail("FKID", interval.pairID);
 				if (e.code() == error_code_transaction_too_old) {
@@ -1507,7 +1507,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 
 		// TraceEvent(SevDebug, interval.end(), data->thisServerID);
 	} catch (Error& e) {
-		// TraceEvent(SevDebug, interval.end(), data->thisServerID).error(e, true).detail("Version", data->version.get());
+		// TraceEvent(SevDebug, interval.end(), data->thisServerID).errorUnsuppressed(e).detail("Version", data->version.get());
 
 		// TODO define the shuttingDown state of cache server
 		if (e.code() == error_code_actor_cancelled &&
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 7f3e2cce75..e3066e9d11 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -1997,8 +1997,8 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 		} catch (Error& e) {
 			self->activePeekStreams--;
 			TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId)
-			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress())
-			    .error(e, true);
+			    .errorUnsuppressed(e)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
@@ -2469,7 +2469,7 @@ ACTOR Future<Void> tLogSnapCreate(TLogSnapRequest snapReq, TLogData* self, Refer
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
-		TraceEvent("TLogExecHelperError").error(e, true /*includeCancelled */);
+		TraceEvent("TLogExecHelperError").errorUnsuppressed(e);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
@@ -3186,7 +3186,7 @@ bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* p
 
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
 	    e.code() == error_code_file_not_found || e.code() == error_code_invalid_cluster_id) {
-		TraceEvent("TLogTerminated", self->dbgid).error(e, true);
+		TraceEvent("TLogTerminated", self->dbgid).errorUnsuppressed(e);
 		return true;
 	} else
 		return false;
@@ -3537,7 +3537,7 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 		}
 	} catch (Error& e) {
 		self.terminated.send(Void());
-		TraceEvent("TLogError", tlogId).error(e, true);
+		TraceEvent("TLogError", tlogId).errorUnsuppressed(e);
 		if (recovered.canBeSet())
 			recovered.send(Void());
 
diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/VFSAsync.cpp
index 17f2b69c9e..950c2f60e9 100644
--- a/fdbserver/VFSAsync.cpp
+++ b/fdbserver/VFSAsync.cpp
@@ -742,7 +742,7 @@ static int asyncSleep(sqlite3_vfs* pVfs, int microseconds) {
 		if (e.isInjectedFault()) {
 			VFSAsyncFile::setInjectedError(SQLITE_ERROR);
 		}
-		TraceEvent(SevError, "VFSAsyncSleepError").error(e, true);
+		TraceEvent(SevError, "VFSAsyncSleepError").errorUnsuppressed(e);
 		return 0;
 	}
 }
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index dde5b8087e..13d9c18d15 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -1181,12 +1181,12 @@ public:
 					             page->getChecksum());
 					Error e = checksum_failed();
 					TraceEvent(SevError, "RedwoodChecksumFailed")
+					    .error(e)
 					    .detail("PageID", c.pageID)
 					    .detail("PageSize", self->pager->getPhysicalPageSize())
 					    .detail("Offset", c.pageID * self->pager->getPhysicalPageSize())
 					    .detail("CalculatedChecksum", page->calculateChecksum(c.pageID))
-					    .detail("ChecksumInPage", page->getChecksum())
-					    .error(e);
+					    .detail("ChecksumInPage", page->getChecksum());
 					throw e;
 				}
 
@@ -1228,9 +1228,9 @@ public:
 					if (entriesRead != self->numEntries) {
 						Error e = internal_error(); // TODO:  Something better?
 						TraceEvent(SevError, "RedwoodQueueNumEntriesMisMatch")
+						    .error(e)
 						    .detail("EntriesRead", entriesRead)
-						    .detail("ExpectedEntries", self->numEntries)
-						    .error(e);
+						    .detail("ExpectedEntries", self->numEntries);
 						throw e;
 					}
 					res.sendError(end_of_stream());
@@ -2245,7 +2245,7 @@ public:
 					}
 
 					Error e = checksum_failed();
-					TraceEvent(SevError, "RedwoodRecoveryFailed").detail("Filename", self->filename).error(e);
+					TraceEvent(SevError, "RedwoodRecoveryFailed").error(e).detail("Filename", self->filename);
 					throw e;
 				}
 				recoveredHeader = true;
@@ -2256,10 +2256,10 @@ public:
 			if (self->pHeader->formatVersion != Header::FORMAT_VERSION) {
 				Error e = unsupported_format_version();
 				TraceEvent(SevWarn, "RedwoodRecoveryFailedWrongVersion")
+				    .error(e)
 				    .detail("Filename", self->filename)
 				    .detail("Version", self->pHeader->formatVersion)
-				    .detail("ExpectedVersion", Header::FORMAT_VERSION)
-				    .error(e);
+				    .detail("ExpectedVersion", Header::FORMAT_VERSION);
 				throw e;
 			}
 
@@ -2904,13 +2904,13 @@ public:
 				if (g_network->isSimulated() && g_simulator.checkInjectedCorruption())
 					e = e.asInjectedFault();
 				TraceEvent(SevError, "RedwoodChecksumFailed")
+				    .error(e)
 				    .detail("Filename", self->filename.c_str())
 				    .detail("PageID", pageID)
 				    .detail("PageSize", self->physicalPageSize)
 				    .detail("Offset", pageID * self->physicalPageSize)
 				    .detail("CalculatedChecksum", page->calculateChecksum(pageID))
-				    .detail("ChecksumInPage", page->getChecksum())
-				    .error(e);
+				    .detail("ChecksumInPage", page->getChecksum());
 				ASSERT(false);
 				throw e;
 			}
@@ -2955,13 +2955,13 @@ public:
 			debug_printf("DWALPager(%s) checksum failed for %s\n", self->filename.c_str(), toString(pageIDs).c_str());
 			Error e = checksum_failed();
 			TraceEvent(SevError, "RedwoodChecksumFailed")
+			    .error(e)
 			    .detail("Filename", self->filename.c_str())
 			    .detail("PageID", pageIDs)
 			    .detail("PageSize", self->physicalPageSize)
 			    .detail("Offset", pageIDs.front() * self->physicalPageSize)
 			    .detail("CalculatedChecksum", page->calculateChecksum(pageIDs.front()))
-			    .detail("ChecksumInPage", page->getChecksum())
-			    .error(e);
+			    .detail("ChecksumInPage", page->getChecksum());
 			ASSERT(false);
 			throw e;
 		}
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 5159ba2243..1a2a91f77b 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -1745,9 +1745,9 @@ int main(int argc, char* argv[]) {
 				} else {
 					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
+					    .error(e)
 					    .detail("Knob", printable(knobName))
-					    .detail("Value", printable(knobValueString))
-					    .error(e);
+					    .detail("Value", printable(knobValueString));
 					throw;
 				}
 			}
diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp
index a495706db4..3f46250498 100644
--- a/fdbserver/networktest.actor.cpp
+++ b/fdbserver/networktest.actor.cpp
@@ -539,8 +539,8 @@ struct P2PNetworkTest {
 		} catch (Error& e) {
 			++self->sessionErrors;
 			TraceEvent(SevError, incoming ? "P2PIncomingSessionError" : "P2POutgoingSessionError")
-			    .detail("Remote", conn->getPeerAddress())
-			    .error(e);
+			    .error(e)
+			    .detail("Remote", conn->getPeerAddress());
 		}
 
 		return Void();
@@ -557,7 +557,7 @@ struct P2PNetworkTest {
 				wait(doSession(self, conn, false));
 			} catch (Error& e) {
 				++self->connectErrors;
-				TraceEvent(SevError, "P2POutgoingError").detail("Remote", remote).error(e);
+				TraceEvent(SevError, "P2POutgoingError").error(e).detail("Remote", remote);
 				wait(delay(1));
 			}
 		}
@@ -575,7 +575,7 @@ struct P2PNetworkTest {
 				sessions.add(doSession(self, conn, true));
 			} catch (Error& e) {
 				++self->acceptErrors;
-				TraceEvent(SevError, "P2PIncomingError").detail("Listener", listener->getListenAddress()).error(e);
+				TraceEvent(SevError, "P2PIncomingError").error(e).detail("Listener", listener->getListenAddress());
 			}
 		}
 	}
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 5f0939b833..a4fed0c584 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -4289,7 +4289,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				}
 				if (nfk == keys.begin) {
 					TraceEvent("FKBlockFail", data->thisServerID)
-					    .error(e, true)
+					    .errorUnsuppressed(e)
 					    .suppressFor(1.0)
 					    .detail("FKID", interval.pairID);
 
@@ -4455,7 +4455,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 		TraceEvent(SevDebug, interval.end(), data->thisServerID);
 	} catch (Error& e) {
-		TraceEvent(SevDebug, interval.end(), data->thisServerID).error(e, true).detail("Version", data->version.get());
+		TraceEvent(SevDebug, interval.end(), data->thisServerID)
+		    .errorUnsuppressed(e)
+		    .detail("Version", data->version.get());
 
 		if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
 			if (shard->phase < AddingShard::Waiting) {
@@ -6842,7 +6844,7 @@ bool storageServerTerminated(StorageServer& self, IKeyValueStore* persistentData
 
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
 	    e.code() == error_code_file_not_found || e.code() == error_code_actor_cancelled) {
-		TraceEvent("StorageServerTerminated", self.thisServerID).error(e, true);
+		TraceEvent("StorageServerTerminated", self.thisServerID).errorUnsuppressed(e);
 		return true;
 	} else
 		return false;
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 1581732625..495f38cf48 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -569,7 +569,7 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
 						throw;
 					TraceEvent(SevError, "TestFailure", workIface.id())
-					    .error(e, true)
+					    .errorUnsuppressed(e)
 					    .detail("Reason", "Error starting workload")
 					    .detail("Workload", workload->description());
 					// ok = false;
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 2dff414e94..326852950f 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -224,7 +224,7 @@ ACTOR Future<Void> handleIOErrors(Future<Void> actor, IClosable* store, UID id,
 				return e.get();
 		}
 		when(ErrorOr<Void> e = wait(storeError)) {
-			TraceEvent("WorkerTerminatingByIOError", id).error(e.getError(), true);
+			TraceEvent("WorkerTerminatingByIOError", id).errorUnsuppressed(e.getError());
 			actor.cancel();
 			// file_not_found can occur due to attempting to open a partially deleted DiskQueue, which should not be
 			// reported SevError.
@@ -1232,7 +1232,7 @@ void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
 	{
 		TraceEvent ev("Role", id);
 		if (e.code() != invalid_error_code)
-			ev.error(e, true);
+			ev.errorUnsuppressed(e);
 		ev.detail("Transition", "End").detail("As", role.roleName).detail("Reason", reason);
 
 		ev.trackLatest(id.shortString() + ".Role");
@@ -1243,7 +1243,7 @@ void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
 
 		TraceEvent err(SevError, type.c_str(), id);
 		if (e.code() != invalid_error_code) {
-			err.error(e, true);
+			err.errorUnsuppressed(e);
 		}
 		err.detail("Reason", reason);
 	}
@@ -1288,7 +1288,7 @@ ACTOR Future<Void> workerSnapCreate(WorkerSnapRequest snapReq, Standalone<String
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
-		TraceEvent("ExecHelperError").error(e, true /*includeCancelled*/);
+		TraceEvent("ExecHelperError").errorUnsuppressed(e);
 		if (e.code() != error_code_operation_cancelled) {
 			snapReq.reply.sendError(e);
 		} else {
diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
index 91c56166a1..1f00eacd22 100644
--- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
+++ b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
@@ -485,7 +485,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					wait(tr2->commit());
 					break;
 				} catch (Error& e) {
-					TraceEvent("DRU_RestoreSetupError").error(e, true);
+					TraceEvent("DRU_RestoreSetupError").errorUnsuppressed(e);
 					wait(tr2->onError(e));
 				}
 			}
diff --git a/fdbserver/workloads/ConfigIncrement.actor.cpp b/fdbserver/workloads/ConfigIncrement.actor.cpp
index 1a50085e58..da15fceba6 100644
--- a/fdbserver/workloads/ConfigIncrement.actor.cpp
+++ b/fdbserver/workloads/ConfigIncrement.actor.cpp
@@ -83,8 +83,8 @@ class ConfigIncrementWorkload : public TestWorkload {
 						break;
 					} catch (Error& e) {
 						TraceEvent(SevDebug, "ConfigIncrementError")
-						    .detail("LastKnownValue", self->lastKnownValue)
-						    .error(e, true /* include cancelled  */);
+						    .errorUnsuppressed(e)
+						    .detail("LastKnownValue", self->lastKnownValue);
 						wait(tr->onError(e));
 						++self->retries;
 					}
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 2ec76d4d62..ef4f1a4599 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1073,11 +1073,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				// If the storage server doesn't reply, then return -1
 				if (!reply.present()) {
 					TraceEvent("ConsistencyCheck_FailedToFetchMetrics")
+					    .error(reply.getError())
 					    .detail("Begin", printable(shard.begin))
 					    .detail("End", printable(shard.end))
 					    .detail("StorageServer", storageServers[i].id())
-					    .detail("IsTSS", storageServers[i].isTss() ? "True" : "False")
-					    .error(reply.getError());
+					    .detail("IsTSS", storageServers[i].isTss() ? "True" : "False");
 					estimatedBytes.push_back(-1);
 				}
 
@@ -1495,6 +1495,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    rangeResult.isError() ? rangeResult.getError() : rangeResult.get().error.get();
 
 								TraceEvent("ConsistencyCheck_StorageServerUnavailable")
+								    .errorUnsuppressed(e)
 								    .suppressFor(1.0)
 								    .detail("StorageServer", storageServers[j])
 								    .detail("ShardBegin", printable(range.begin))
@@ -1503,8 +1504,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail("UID", storageServerInterfaces[j].id())
 								    .detail("GetKeyValuesToken",
 								            storageServerInterfaces[j].getKeyValues.getEndpoint().token)
-								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False")
-								    .error(e);
+								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
 
 								// All shards should be available in quiscence
 								if (self->performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp
index 0973a2e61c..ad978b6f7e 100644
--- a/fdbserver/workloads/DiskFailureInjection.actor.cpp
+++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp
@@ -106,8 +106,8 @@ struct DiskFailureInjectionWorkload : TestWorkload {
 				    .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString());
 			} else {
 				TraceEvent(SevError, "DiskFailureInjectionFailed")
-				    .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString())
-				    .error(err);
+				    .error(err)
+				    .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString());
 			}
 		}
 	}
diff --git a/fdbserver/workloads/FastTriggeredWatches.actor.cpp b/fdbserver/workloads/FastTriggeredWatches.actor.cpp
index c91203fdda..0cd0ed294d 100644
--- a/fdbserver/workloads/FastTriggeredWatches.actor.cpp
+++ b/fdbserver/workloads/FastTriggeredWatches.actor.cpp
@@ -149,7 +149,7 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
 			}
 			return Void();
 		} catch (Error& e) {
-			TraceEvent(SevError, "FastWatchError").error(e, true);
+			TraceEvent(SevError, "FastWatchError").errorUnsuppressed(e);
 			throw;
 		}
 	}
diff --git a/fdbserver/workloads/LowLatency.actor.cpp b/fdbserver/workloads/LowLatency.actor.cpp
index 74fb738415..800da48e05 100644
--- a/fdbserver/workloads/LowLatency.actor.cpp
+++ b/fdbserver/workloads/LowLatency.actor.cpp
@@ -88,7 +88,7 @@ struct LowLatencyWorkload : TestWorkload {
 						}
 						break;
 					} catch (Error& e) {
-						TraceEvent("LowLatencyTransactionFailed").error(e, true);
+						TraceEvent("LowLatencyTransactionFailed").errorUnsuppressed(e);
 						wait(tr.onError(e));
 						++self->retries;
 					}
@@ -105,7 +105,7 @@ struct LowLatencyWorkload : TestWorkload {
 			}
 			return Void();
 		} catch (Error& e) {
-			TraceEvent(SevError, "LowLatencyError").error(e, true);
+			TraceEvent(SevError, "LowLatencyError").errorUnsuppressed(e);
 			throw;
 		}
 	}
diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp
index 9e32b7aa37..6864ffe5e2 100644
--- a/fdbserver/workloads/Mako.actor.cpp
+++ b/fdbserver/workloads/Mako.actor.cpp
@@ -815,7 +815,7 @@ struct MakoWorkload : TestWorkload {
 				}
 				return true;
 			} catch (Error& e) {
-				TraceEvent("FailedToCalculateChecksum").detail("ChecksumIndex", csIdx).error(e);
+				TraceEvent("FailedToCalculateChecksum").error(e).detail("ChecksumIndex", csIdx);
 				wait(tr.onError(e));
 			}
 		}
diff --git a/fdbserver/workloads/Performance.actor.cpp b/fdbserver/workloads/Performance.actor.cpp
index 99a35b1309..77d7a45f43 100644
--- a/fdbserver/workloads/Performance.actor.cpp
+++ b/fdbserver/workloads/Performance.actor.cpp
@@ -175,7 +175,9 @@ struct PerformanceWorkload : TestWorkload {
 				DistributedTestResults r = wait(runWorkload(cx, self->testers, spec));
 				results = r;
 			} catch (Error& e) {
-				TraceEvent("PerformanceRunError").error(e, true).detail("Workload", printable(self->probeWorkload));
+				TraceEvent("PerformanceRunError")
+				    .errorUnsuppressed(e)
+				    .detail("Workload", printable(self->probeWorkload));
 				break;
 			}
 			PerfMetric tpsMetric = self->getNamedMetric("Transactions/sec", results.metrics);
diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.actor.cpp
index f669aed463..02a03e17a8 100644
--- a/fdbserver/workloads/RandomMoveKeys.actor.cpp
+++ b/fdbserver/workloads/RandomMoveKeys.actor.cpp
@@ -154,7 +154,7 @@ struct MoveKeysWorkload : TestWorkload {
 			TraceEvent(relocateShardInterval.end()).detail("Result", "Success");
 			return Void();
 		} catch (Error& e) {
-			TraceEvent(relocateShardInterval.end(), self->dbInfo->get().master.id()).error(e, true);
+			TraceEvent(relocateShardInterval.end(), self->dbInfo->get().master.id()).errorUnsuppressed(e);
 			throw;
 		}
 	}
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index 6b8cfe1553..d75ca27f14 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -668,7 +668,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
 				ASSERT(valueObj["command"].get_str() == "exclude" && !valueObj["retriable"].get_bool());
 			} else {
-				TraceEvent(SevDebug, "UnexpectedError").detail("Command", "Exclude").error(e);
+				TraceEvent(SevDebug, "UnexpectedError").error(e).detail("Command", "Exclude");
 				wait(tx->onError(e));
 			}
 			tx->reset();
@@ -737,7 +737,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
 					ASSERT(valueObj["command"].get_str() == "setclass" && !valueObj["retriable"].get_bool());
 				} else {
-					TraceEvent(SevDebug, "UnexpectedError").detail("Command", "Setclass").error(e);
+					TraceEvent(SevDebug, "UnexpectedError").error(e).detail("Command", "Setclass");
 					wait(tx->onError(e));
 				}
 				tx->reset();
diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp
index 1c1c0ad6d2..cffd6aaad3 100644
--- a/fdbserver/workloads/UnitTests.actor.cpp
+++ b/fdbserver/workloads/UnitTests.actor.cpp
@@ -149,7 +149,7 @@ struct UnitTestWorkload : TestWorkload {
 			self->totalWallTime += wallTime;
 			self->totalSimTime += simTime;
 			TraceEvent(result.code() != error_code_success ? SevError : SevInfo, "UnitTest")
-			    .error(result, true)
+			    .errorUnsuppressed(result)
 			    .detail("Name", test->name)
 			    .detail("File", test->file)
 			    .detail("Line", test->line)
diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index e0d47d68e4..d18b6d0fd9 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -1055,7 +1055,7 @@ ACTOR Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self,
 			watches.clear();
 			self->changeCount.insert(allKeys, 0);
 			doingCommit = false;
-			//TraceEvent("WDRError").error(e, true);
+			//TraceEvent("WDRError").errorUnsuppressed(e);
 			if (e.code() == error_code_database_locked) {
 				self->memoryDatabase = self->lastCommittedDatabase;
 				self->addedConflicts.insert(allKeys, false);
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index aabba21917..084eaa5039 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -592,7 +592,7 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) {
 	struct statvfs buf;
 	if (statvfs(directory.c_str(), &buf)) {
 		Error e = systemErrorCodeToError();
-		TraceEvent(SevError, "GetDiskBytesStatvfsError").detail("Directory", directory).GetLastError().error(e);
+		TraceEvent(SevError, "GetDiskBytesStatvfsError").error(e).detail("Directory", directory).GetLastError();
 		throw e;
 	}
 
@@ -601,7 +601,7 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) {
 	struct statfs buf;
 	if (statfs(directory.c_str(), &buf)) {
 		Error e = systemErrorCodeToError();
-		TraceEvent(SevError, "GetDiskBytesStatfsError").detail("Directory", directory).GetLastError().error(e);
+		TraceEvent(SevError, "GetDiskBytesStatfsError").error(e).detail("Directory", directory).GetLastError();
 		throw e;
 	}
 
@@ -1221,7 +1221,7 @@ void getDiskStatistics(std::string const& directory,
 	struct statfs buf;
 	if (statfs(directory.c_str(), &buf)) {
 		Error e = systemErrorCodeToError();
-		TraceEvent(SevError, "GetDiskStatisticsStatfsError").detail("Directory", directory).GetLastError().error(e);
+		TraceEvent(SevError, "GetDiskStatisticsStatfsError").error(e).detail("Directory", directory).GetLastError();
 		throw e;
 	}
 
@@ -2325,7 +2325,7 @@ bool deleteFile(std::string const& filename) {
 #error Port me!
 #endif
 	Error e = systemErrorCodeToError();
-	TraceEvent(SevError, "DeleteFile").detail("Filename", filename).GetLastError().error(e);
+	TraceEvent(SevError, "DeleteFile").error(e).detail("Filename", filename).GetLastError();
 	throw e;
 }
 
@@ -2381,10 +2381,10 @@ bool createDirectory(std::string const& directory) {
 			}
 
 			TraceEvent(SevError, "CreateDirectory")
+			    .error(e)
 			    .detail("Directory", directory)
 			    .detailf("UnixErrorCode", "%x", errno)
-			    .detail("UnixError", strerror(mkdirErrno))
-			    .error(e);
+			    .detail("UnixError", strerror(mkdirErrno));
 			throw e;
 		}
 		createdDirectory();
@@ -2471,7 +2471,7 @@ std::string abspath(std::string const& path, bool resolveLinks, bool mustExist)
 	if (path.empty()) {
 		Error e = platform_error();
 		Severity sev = e.code() == error_code_io_error ? SevError : SevWarnAlways;
-		TraceEvent(sev, "AbsolutePathError").detail("Path", path).error(e);
+		TraceEvent(sev, "AbsolutePathError").error(e).detail("Path", path);
 		throw e;
 	}
 
@@ -2489,7 +2489,7 @@ std::string abspath(std::string const& path, bool resolveLinks, bool mustExist)
 		if (mustExist && !fileExists(clean)) {
 			Error e = systemErrorCodeToError();
 			Severity sev = e.code() == error_code_io_error ? SevError : SevWarnAlways;
-			TraceEvent(sev, "AbsolutePathError").detail("Path", path).GetLastError().error(e);
+			TraceEvent(sev, "AbsolutePathError").error(e).detail("Path", path).GetLastError();
 			throw e;
 		}
 		return clean;
@@ -2527,7 +2527,7 @@ std::string abspath(std::string const& path, bool resolveLinks, bool mustExist)
 		}
 		Error e = systemErrorCodeToError();
 		Severity sev = e.code() == error_code_io_error ? SevError : SevWarnAlways;
-		TraceEvent(sev, "AbsolutePathError").detail("Path", path).GetLastError().error(e);
+		TraceEvent(sev, "AbsolutePathError").error(e).detail("Path", path).GetLastError();
 		throw e;
 	}
 	return std::string(r);
@@ -3177,7 +3177,7 @@ int eraseDirectoryRecursive(std::string const& dir) {
 	   place */
 	if (error && errno != ENOENT) {
 		Error e = systemErrorCodeToError();
-		TraceEvent(SevError, "EraseDirectoryRecursiveError").detail("Directory", dir).GetLastError().error(e);
+		TraceEvent(SevError, "EraseDirectoryRecursiveError").error(e).detail("Directory", dir).GetLastError();
 		throw e;
 	}
 #else
diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h
index 83495b2121..3b5315753a 100644
--- a/flow/ThreadHelper.actor.h
+++ b/flow/ThreadHelper.actor.h
@@ -653,7 +653,7 @@ Future<Void> doOnMainThread(Future<Void> signal, F f, ThreadSingleAssignmentVar<
 		result->send(r);
 	} catch (Error& e) {
 		if (!result->canBeSet()) {
-			TraceEvent(SevError, "OnMainThreadSetTwice").error(e, true);
+			TraceEvent(SevError, "OnMainThreadSetTwice").errorUnsuppressed(e);
 		}
 		result->sendError(e);
 	}
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 7aabc10a60..d8a04273ea 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -802,9 +802,12 @@ void addUniversalTraceField(const std::string& name, const std::string& value) {
 	g_traceLog.addUniversalTraceField(name, value);
 }
 
-TraceEvent::TraceEvent() : initialized(true), enabled(false), logged(true) {}
+BaseTraceEvent::BaseTraceEvent() : initialized(true), enabled(false), logged(true) {}
+BaseTraceEvent::BaseTraceEvent(Severity severity, const char* type, UID id)
+  : initialized(false), enabled(g_network == nullptr || FLOW_KNOBS->MIN_TRACE_SEVERITY <= severity), logged(false),
+    severity(severity), type(type), id(id) {}
 
-TraceEvent::TraceEvent(TraceEvent&& ev) {
+BaseTraceEvent::BaseTraceEvent(BaseTraceEvent&& ev) {
 	enabled = ev.enabled;
 	err = ev.err;
 	fields = std::move(ev.fields);
@@ -830,7 +833,7 @@ TraceEvent::TraceEvent(TraceEvent&& ev) {
 	ev.logged = true;
 }
 
-TraceEvent& TraceEvent::operator=(TraceEvent&& ev) {
+BaseTraceEvent& BaseTraceEvent::operator=(BaseTraceEvent&& ev) {
 	// Note: still broken if ev and this are the same memory address.
 	enabled = ev.enabled;
 	err = ev.err;
@@ -867,36 +870,29 @@ Future<Void> pingTraceLogWriterThread() {
 	return g_traceLog.pingWriterThread();
 }
 
-TraceEvent::TraceEvent(const char* type, UID id)
-  : initialized(false), enabled(true), logged(false), severity(SevInfo), type(type), id(id) {
+TraceEvent::TraceEvent(const char* type, UID id) : BaseTraceEvent(SevInfo, type, id) {
 	setMaxFieldLength(0);
 	setMaxEventLength(0);
 }
-TraceEvent::TraceEvent(Severity severity, const char* type, UID id)
-  : initialized(false), enabled(g_network == nullptr || FLOW_KNOBS->MIN_TRACE_SEVERITY <= severity), logged(false),
-    severity(severity), type(type), id(id) {
+TraceEvent::TraceEvent(Severity severity, const char* type, UID id) : BaseTraceEvent(severity, type, id) {
 	setMaxFieldLength(0);
 	setMaxEventLength(0);
 }
-TraceEvent::TraceEvent(TraceInterval& interval, UID id)
-  : initialized(false), enabled(g_network == nullptr || FLOW_KNOBS->MIN_TRACE_SEVERITY <= interval.severity),
-    logged(false), severity(interval.severity), type(interval.type), id(id) {
+TraceEvent::TraceEvent(TraceInterval& interval, UID id) : BaseTraceEvent(interval.severity, interval.type, id) {
 	setMaxFieldLength(0);
 	setMaxEventLength(0);
 
 	init(interval);
 }
 TraceEvent::TraceEvent(Severity severity, TraceInterval& interval, UID id)
-  : initialized(false), enabled(g_network == nullptr || FLOW_KNOBS->MIN_TRACE_SEVERITY <= severity), logged(false),
-    severity(severity), type(interval.type), id(id) {
-
+  : BaseTraceEvent(severity, interval.type, id) {
 	setMaxFieldLength(0);
 	setMaxEventLength(0);
 
 	init(interval);
 }
 
-bool TraceEvent::init(TraceInterval& interval) {
+bool BaseTraceEvent::init(TraceInterval& interval) {
 	bool result = init();
 	switch (interval.count++) {
 	case 0: {
@@ -913,7 +909,7 @@ bool TraceEvent::init(TraceInterval& interval) {
 	return result;
 }
 
-bool TraceEvent::init() {
+bool BaseTraceEvent::init() {
 	ASSERT(!logged);
 	if (initialized) {
 		return enabled;
@@ -1007,7 +1003,7 @@ TraceEvent& TraceEvent::errorImpl(class Error const& error, bool includeCancelle
 	return *this;
 }
 
-TraceEvent& TraceEvent::detailImpl(std::string&& key, std::string&& value, bool writeEventMetricField) {
+BaseTraceEvent& BaseTraceEvent::detailImpl(std::string&& key, std::string&& value, bool writeEventMetricField) {
 	init();
 	if (enabled) {
 		++g_allocation_tracing_disabled;
@@ -1032,25 +1028,25 @@ TraceEvent& TraceEvent::detailImpl(std::string&& key, std::string&& value, bool
 	return *this;
 }
 
-void TraceEvent::setField(const char* key, int64_t value) {
+void BaseTraceEvent::setField(const char* key, int64_t value) {
 	++g_allocation_tracing_disabled;
 	tmpEventMetric->setField(key, value);
 	--g_allocation_tracing_disabled;
 }
 
-void TraceEvent::setField(const char* key, double value) {
+void BaseTraceEvent::setField(const char* key, double value) {
 	++g_allocation_tracing_disabled;
 	tmpEventMetric->setField(key, value);
 	--g_allocation_tracing_disabled;
 }
 
-void TraceEvent::setField(const char* key, const std::string& value) {
+void BaseTraceEvent::setField(const char* key, const std::string& value) {
 	++g_allocation_tracing_disabled;
 	tmpEventMetric->setField(key, Standalone<StringRef>(value));
 	--g_allocation_tracing_disabled;
 }
 
-TraceEvent& TraceEvent::detailf(std::string key, const char* valueFormat, ...) {
+BaseTraceEvent& BaseTraceEvent::detailf(std::string key, const char* valueFormat, ...) {
 	if (enabled) {
 		va_list args;
 		va_start(args, valueFormat);
@@ -1063,7 +1059,7 @@ TraceEvent& TraceEvent::detailf(std::string key, const char* valueFormat, ...) {
 	}
 	return *this;
 }
-TraceEvent& TraceEvent::detailfNoMetric(std::string&& key, const char* valueFormat, ...) {
+BaseTraceEvent& BaseTraceEvent::detailfNoMetric(std::string&& key, const char* valueFormat, ...) {
 	if (enabled) {
 		va_list args;
 		va_start(args, valueFormat);
@@ -1080,15 +1076,14 @@ TraceEvent& TraceEvent::detailfNoMetric(std::string&& key, const char* valueForm
 	return *this;
 }
 
-TraceEvent& TraceEvent::trackLatest(const std::string& trackingKey) {
+BaseTraceEvent& BaseTraceEvent::trackLatest(const std::string& trackingKey) {
 	ASSERT(!logged);
 	this->trackingKey = trackingKey;
 	ASSERT(this->trackingKey.size() != 0 && this->trackingKey[0] != '/' && this->trackingKey[0] != '\\');
 	return *this;
 }
 
-TraceEvent& TraceEvent::sample(double sampleRate, bool logSampleRate) {
-	ASSERT(!logged);
+BaseTraceEvent& TraceEvent::sample(double sampleRate, bool logSampleRate) {
 	if (enabled) {
 		if (initialized) {
 			TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways,
@@ -1107,7 +1102,7 @@ TraceEvent& TraceEvent::sample(double sampleRate, bool logSampleRate) {
 	return *this;
 }
 
-TraceEvent& TraceEvent::suppressFor(double duration, bool logSuppressedEventCount) {
+BaseTraceEvent& TraceEvent::suppressFor(double duration, bool logSuppressedEventCount) {
 	ASSERT(!logged);
 	if (enabled) {
 		if (initialized) {
@@ -1126,23 +1121,25 @@ TraceEvent& TraceEvent::suppressFor(double duration, bool logSuppressedEventCoun
 				}
 			} else {
 				TraceEvent(SevWarnAlways, "SuppressionFromNonNetworkThread").detail("Event", type);
-				detail("__InvalidSuppression__",
-				       ""); // Choosing a detail name that is unlikely to collide with other names
+				// Choosing a detail name that is unlikely to collide with other names
+				detail("__InvalidSuppression__", "");
 			}
 		}
-		init(); // we do not want any future calls on this trace event to disable it, because we have already counted it
-		        // towards our suppression budget
+
+		// we do not want any future calls on this trace event to disable it, because we have already counted it
+		// towards our suppression budget
+		init();
 	}
 
 	return *this;
 }
 
-TraceEvent& TraceEvent::setErrorKind(ErrorKind errorKind) {
+BaseTraceEvent& BaseTraceEvent::setErrorKind(ErrorKind errorKind) {
 	this->errorKind = errorKind;
 	return *this;
 }
 
-TraceEvent& TraceEvent::setMaxFieldLength(int maxFieldLength) {
+BaseTraceEvent& BaseTraceEvent::setMaxFieldLength(int maxFieldLength) {
 	ASSERT(!logged);
 	if (maxFieldLength == 0) {
 		this->maxFieldLength = FLOW_KNOBS ? FLOW_KNOBS->MAX_TRACE_FIELD_LENGTH : 495;
@@ -1165,15 +1162,15 @@ uint64_t getTraceThreadId() {
 	return threadId;
 }
 
-void TraceEvent::setThreadId() {
+void BaseTraceEvent::setThreadId() {
 	this->detail("ThreadID", getTraceThreadId());
 }
 
-int TraceEvent::getMaxFieldLength() const {
+int BaseTraceEvent::getMaxFieldLength() const {
 	return maxFieldLength;
 }
 
-TraceEvent& TraceEvent::setMaxEventLength(int maxEventLength) {
+BaseTraceEvent& BaseTraceEvent::setMaxEventLength(int maxEventLength) {
 	ASSERT(!logged);
 	if (maxEventLength == 0) {
 		this->maxEventLength = FLOW_KNOBS ? FLOW_KNOBS->MAX_TRACE_EVENT_LENGTH : 4000;
@@ -1184,11 +1181,11 @@ TraceEvent& TraceEvent::setMaxEventLength(int maxEventLength) {
 	return *this;
 }
 
-int TraceEvent::getMaxEventLength() const {
+int BaseTraceEvent::getMaxEventLength() const {
 	return maxEventLength;
 }
 
-TraceEvent& TraceEvent::GetLastError() {
+BaseTraceEvent& BaseTraceEvent::GetLastError() {
 #ifdef _WIN32
 	return detailf("WinErrorCode", "%x", ::GetLastError());
 #elif defined(__unixish__)
@@ -1196,21 +1193,21 @@ TraceEvent& TraceEvent::GetLastError() {
 #endif
 }
 
-unsigned long TraceEvent::eventCounts[NUM_MAJOR_LEVELS_OF_EVENTS] = { 0, 0, 0, 0, 0 };
+unsigned long BaseTraceEvent::eventCounts[NUM_MAJOR_LEVELS_OF_EVENTS] = { 0, 0, 0, 0, 0 };
 
-unsigned long TraceEvent::CountEventsLoggedAt(Severity sev) {
+unsigned long BaseTraceEvent::CountEventsLoggedAt(Severity sev) {
 	ASSERT(sev <= SevMaxUsed);
 	return TraceEvent::eventCounts[sev / 10];
 }
 
-TraceEvent& TraceEvent::backtrace(const std::string& prefix) {
+BaseTraceEvent& BaseTraceEvent::backtrace(const std::string& prefix) {
 	ASSERT(!logged);
 	if (this->severity == SevError || !enabled)
 		return *this; // We'll backtrace this later in ~TraceEvent
 	return detail(prefix + "Backtrace", platform::get_backtrace());
 }
 
-void TraceEvent::log() {
+void BaseTraceEvent::log() {
 	if (!logged) {
 		init();
 		++g_allocation_tracing_disabled;
@@ -1256,7 +1253,7 @@ void TraceEvent::log() {
 				}
 			}
 		} catch (Error& e) {
-			TraceEvent(SevError, "TraceEventLoggingError").error(e, true);
+			TraceEvent(SevError, "TraceEventLoggingError").errorUnsuppressed(e);
 		}
 		tmpEventMetric.reset();
 		logged = true;
@@ -1264,13 +1261,13 @@ void TraceEvent::log() {
 	}
 }
 
-TraceEvent::~TraceEvent() {
+BaseTraceEvent::~BaseTraceEvent() {
 	log();
 }
 
-thread_local bool TraceEvent::networkThread = false;
+thread_local bool BaseTraceEvent::networkThread = false;
 
-void TraceEvent::setNetworkThread() {
+void BaseTraceEvent::setNetworkThread() {
 	if (!networkThread) {
 		if (FLOW_KNOBS->ALLOCATION_TRACING_ENABLED) {
 			--g_allocation_tracing_disabled;
@@ -1282,11 +1279,11 @@ void TraceEvent::setNetworkThread() {
 	}
 }
 
-bool TraceEvent::isNetworkThread() {
+bool BaseTraceEvent::isNetworkThread() {
 	return networkThread;
 }
 
-double TraceEvent::getCurrentTime() {
+double BaseTraceEvent::getCurrentTime() {
 	if (g_trace_clock.load() == TRACE_CLOCK_NOW) {
 		if (!isNetworkThread() || !g_network) {
 			return timer_monotonic();
@@ -1303,7 +1300,7 @@ double TraceEvent::getCurrentTime() {
 // This only has second-resolution for the simple reason
 // that std::put_time does not support higher resolution.
 // This is fine since we always log the flow time as well.
-std::string TraceEvent::printRealTime(double time) {
+std::string BaseTraceEvent::printRealTime(double time) {
 	using Clock = std::chrono::system_clock;
 	time_t ts = Clock::to_time_t(Clock::time_point(
 	    std::chrono::duration_cast<Clock::duration>(std::chrono::duration<double, std::ratio<1>>(time))));
diff --git a/flow/Trace.h b/flow/Trace.h
index b30b5405de..1e60d6049f 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -384,15 +384,12 @@ struct SpecialTraceMetricType
 
 TRACE_METRIC_TYPE(double, double);
 
-struct TraceEvent {
-	TraceEvent();
-	TraceEvent(const char* type, UID id = UID()); // Assumes SevInfo severity
-	TraceEvent(Severity, const char* type, UID id = UID());
-	TraceEvent(struct TraceInterval&, UID id = UID());
-	TraceEvent(Severity severity, struct TraceInterval& interval, UID id = UID());
+struct BaseTraceEvent {
+	BaseTraceEvent();
+	BaseTraceEvent(Severity, const char* type, UID id = UID());
 
-	TraceEvent(TraceEvent&& ev);
-	TraceEvent& operator=(TraceEvent&& ev);
+	BaseTraceEvent(BaseTraceEvent&& ev);
+	BaseTraceEvent& operator=(BaseTraceEvent&& ev);
 
 	static void setNetworkThread();
 	static bool isNetworkThread();
@@ -400,16 +397,8 @@ struct TraceEvent {
 	static double getCurrentTime();
 	static std::string printRealTime(double time);
 
-	// Must be called directly after constructing the trace event
-	TraceEvent& error(const class Error& e, bool includeCancelled = false) {
-		if (enabled) {
-			return errorImpl(e, includeCancelled);
-		}
-		return *this;
-	}
-
 	template <class T>
-	typename std::enable_if<Traceable<T>::value, TraceEvent&>::type detail(std::string&& key, const T& value) {
+	typename std::enable_if<Traceable<T>::value, BaseTraceEvent&>::type detail(std::string&& key, const T& value) {
 		if (enabled && init()) {
 			auto s = Traceable<T>::toString(value);
 			addMetric(key.c_str(), value, s);
@@ -419,7 +408,7 @@ struct TraceEvent {
 	}
 
 	template <class T>
-	typename std::enable_if<Traceable<T>::value, TraceEvent&>::type detail(const char* key, const T& value) {
+	typename std::enable_if<Traceable<T>::value, BaseTraceEvent&>::type detail(const char* key, const T& value) {
 		if (enabled && init()) {
 			auto s = Traceable<T>::toString(value);
 			addMetric(key, value, s);
@@ -428,16 +417,16 @@ struct TraceEvent {
 		return *this;
 	}
 	template <class T>
-	typename std::enable_if<std::is_enum<T>::value, TraceEvent&>::type detail(const char* key, T value) {
+	typename std::enable_if<std::is_enum<T>::value, BaseTraceEvent&>::type detail(const char* key, T value) {
 		if (enabled && init()) {
 			setField(key, int64_t(value));
 			return detailImpl(std::string(key), format("%d", value), false);
 		}
 		return *this;
 	}
-	TraceEvent& detailf(std::string key, const char* valueFormat, ...);
+	BaseTraceEvent& detailf(std::string key, const char* valueFormat, ...);
 
-private:
+protected:
 	template <class T>
 	typename std::enable_if<SpecialTraceMetricType<T>::value, void>::type addMetric(const char* key,
 	                                                                                const T& value,
@@ -457,38 +446,32 @@ private:
 	void setField(const char* key, const std::string& value);
 	void setThreadId();
 
-	TraceEvent& errorImpl(const class Error& e, bool includeCancelled = false);
 	// Private version of detailf that does NOT write to the eventMetric.  This is to be used by other detail methods
 	// which can write field metrics of a more appropriate type than string but use detailf() to add to the TraceEvent.
-	TraceEvent& detailfNoMetric(std::string&& key, const char* valueFormat, ...);
-	TraceEvent& detailImpl(std::string&& key, std::string&& value, bool writeEventMetricField = true);
+	BaseTraceEvent& detailfNoMetric(std::string&& key, const char* valueFormat, ...);
+	BaseTraceEvent& detailImpl(std::string&& key, std::string&& value, bool writeEventMetricField = true);
 
 public:
-	TraceEvent& backtrace(const std::string& prefix = "");
-	TraceEvent& trackLatest(const std::string& trackingKey);
-	TraceEvent& sample(double sampleRate, bool logSampleRate = true);
-
+	BaseTraceEvent& backtrace(const std::string& prefix = "");
+	BaseTraceEvent& trackLatest(const std::string& trackingKey);
 	// Sets the maximum length a field can be before it gets truncated. A value of 0 uses the default, a negative value
 	// disables truncation. This should be called before the field whose length you want to change, and it can be
 	// changed multiple times in a single event.
-	TraceEvent& setMaxFieldLength(int maxFieldLength);
+	BaseTraceEvent& setMaxFieldLength(int maxFieldLength);
 
 	int getMaxFieldLength() const;
 
 	// Sets the maximum event length before the event gets suppressed and a warning is logged. A value of 0 uses the
 	// default, a negative value disables length suppression. This should be called before adding details.
-	TraceEvent& setMaxEventLength(int maxEventLength);
+	BaseTraceEvent& setMaxEventLength(int maxEventLength);
 
 	int getMaxEventLength() const;
 
-	// Cannot call other functions which could disable the trace event afterwords
-	TraceEvent& suppressFor(double duration, bool logSuppressedEventCount = true);
-
-	TraceEvent& GetLastError();
+	BaseTraceEvent& GetLastError();
 
 	bool isEnabled() const { return enabled; }
 
-	TraceEvent& setErrorKind(ErrorKind errorKind);
+	BaseTraceEvent& setErrorKind(ErrorKind errorKind);
 
 	explicit operator bool() const { return enabled; }
 
@@ -496,7 +479,7 @@ public:
 
 	void disable() { enabled = false; } // Disables the trace event so it doesn't get
 
-	~TraceEvent(); // Actually logs the event
+	virtual ~BaseTraceEvent(); // Actually logs the event
 
 	// Return the number of invocations of TraceEvent() at the specified logging level.
 	static unsigned long CountEventsLoggedAt(Severity);
@@ -505,7 +488,7 @@ public:
 
 	const TraceEventFields& getFields() const { return fields; }
 
-private:
+protected:
 	bool initialized;
 	bool enabled;
 	bool logged;
@@ -531,6 +514,34 @@ private:
 	bool init(struct TraceInterval&);
 };
 
+struct TraceEvent : public BaseTraceEvent {
+	TraceEvent() {}
+	TraceEvent(const char* type, UID id = UID()); // Assumes SevInfo severity
+	TraceEvent(Severity, const char* type, UID id = UID());
+	TraceEvent(struct TraceInterval&, UID id = UID());
+	TraceEvent(Severity severity, struct TraceInterval& interval, UID id = UID());
+
+	BaseTraceEvent& error(const class Error& e) {
+		if (enabled) {
+			return errorImpl(e, false);
+		}
+		return *this;
+	}
+
+	TraceEvent& errorUnsuppressed(const class Error& e) {
+		if (enabled) {
+			return errorImpl(e, true);
+		}
+		return *this;
+	}
+
+	BaseTraceEvent& sample(double sampleRate, bool logSampleRate = true);
+	BaseTraceEvent& suppressFor(double duration, bool logSuppressedEventCount = true);
+
+private:
+	TraceEvent& errorImpl(const class Error& e, bool includeCancelled = false);
+};
+
 class StringRef;
 
 struct TraceInterval {
diff --git a/flow/flow.h b/flow/flow.h
index 86d6ed3042..6f40a2be8b 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -61,7 +61,7 @@
 #define TEST(condition)                                                                                                \
 	if (!(condition)) {                                                                                                \
 	} else {                                                                                                           \
-		static TraceEvent* __test =                                                                                    \
+		static BaseTraceEvent* __test =                                                                                \
 		    &(TraceEvent(intToSeverity(FLOW_KNOBS->CODE_COV_TRACE_EVENT_SEVERITY), "CodeCoverage")                     \
 		          .detail("File", __FILE__)                                                                            \
 		          .detail("Line", __LINE__)                                                                            \
diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h
index 23bc5da602..8d1465b756 100644
--- a/flow/genericactors.actor.h
+++ b/flow/genericactors.actor.h
@@ -50,7 +50,7 @@ Future<T> traceAfter(Future<T> what, const char* type, const char* key, X value,
 		return val;
 	} catch (Error& e) {
 		if (traceErrors)
-			TraceEvent(type).error(e, true).detail(key, value);
+			TraceEvent(type).errorUnsuppressed(e).detail(key, value);
 		throw;
 	}
 }
@@ -67,7 +67,7 @@ Future<T> traceAfterCall(Future<T> what, const char* type, const char* key, X fu
 		return val;
 	} catch (Error& e) {
 		if (traceErrors)
-			TraceEvent(type).error(e, true);
+			TraceEvent(type).errorUnsuppressed(e);
 		throw;
 	}
 }

From e7d2abd764151c4bcf23815e2215022f9b2cdaef Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Thu, 24 Feb 2022 12:28:13 -0800
Subject: [PATCH 110/210] Fix formatting issue

---
 fdbserver/MutationTracking.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp
index eec751cbea..0acae5cdac 100644
--- a/fdbserver/MutationTracking.cpp
+++ b/fdbserver/MutationTracking.cpp
@@ -41,7 +41,7 @@ TraceEvent debugMutationEnabled(const char* context, Version version, MutationRe
 
 	for (auto& labelKey : debugKeys) {
 		if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
-		    KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
+		     KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
 		    mutation.param1 == labelKey.second) {
 			label = labelKey.first;
 			break;

From d3be393246599caaa0eb7321f54e81f07c4dab30 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Thu, 24 Feb 2022 13:29:44 -0800
Subject: [PATCH 111/210] Add comments to BaseTraceEvent and TraceEvent. Also
 make BaseTraceEvent contstructor protected.

---
 flow/Trace.h | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/flow/Trace.h b/flow/Trace.h
index 1e60d6049f..cf89ce6b18 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -384,10 +384,13 @@ struct SpecialTraceMetricType
 
 TRACE_METRIC_TYPE(double, double);
 
+// The BaseTraceEvent class is the parent class of TraceEvent and provides all functionality on the TraceEvent except
+// for the functionality that can be used to suppress the trace event.
+//
+// This class is not intended to be used directly. Instead, this type is returned from most calls on trace events
+// (e.g. detail). This is done to disallow calling suppression functions anywhere but first in a chained sequence of
+// trace event function calls.
 struct BaseTraceEvent {
-	BaseTraceEvent();
-	BaseTraceEvent(Severity, const char* type, UID id = UID());
-
 	BaseTraceEvent(BaseTraceEvent&& ev);
 	BaseTraceEvent& operator=(BaseTraceEvent&& ev);
 
@@ -427,6 +430,9 @@ struct BaseTraceEvent {
 	BaseTraceEvent& detailf(std::string key, const char* valueFormat, ...);
 
 protected:
+	BaseTraceEvent();
+	BaseTraceEvent(Severity, const char* type, UID id = UID());
+
 	template <class T>
 	typename std::enable_if<SpecialTraceMetricType<T>::value, void>::type addMetric(const char* key,
 	                                                                                const T& value,
@@ -514,6 +520,8 @@ protected:
 	bool init(struct TraceInterval&);
 };
 
+// The TraceEvent class provides the implementation for BaseTraceEvent. The only functions that should be implemented
+// here are those that must be called first in a trace event call sequence, such as the suppression functions.
 struct TraceEvent : public BaseTraceEvent {
 	TraceEvent() {}
 	TraceEvent(const char* type, UID id = UID()); // Assumes SevInfo severity

From 9c99c59b0ca9e3fac6eefeff04be7316556e87ad Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 23 Feb 2022 14:32:19 -0800
Subject: [PATCH 112/210] Fix get key location overloading proxies

---
 fdbclient/ClientKnobs.cpp     |  2 +
 fdbclient/ClientKnobs.h       |  2 +
 fdbclient/DatabaseContext.h   | 14 ++++++
 fdbclient/NativeAPI.actor.cpp | 88 +++++++++++++++++++++++++++++++----
 4 files changed, 97 insertions(+), 9 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 6b896cca85..ae006a4d02 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -86,6 +86,8 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( LOCATION_CACHE_EVICTION_SIZE,         600000 );
 	init( LOCATION_CACHE_EVICTION_SIZE_SIM,         10 ); if( randomize && BUGGIFY ) LOCATION_CACHE_EVICTION_SIZE_SIM = 3;
+	init( LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD,     60 );
+	init( LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL,    60 );
 
 	init( GET_RANGE_SHARD_LIMIT,                     2 );
 	init( WARM_RANGE_SHARD_LIMIT,                  100 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 43dec60277..a11e67a00f 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -86,6 +86,8 @@ public:
 	// When locationCache in DatabaseContext gets to be this size, items will be evicted
 	int LOCATION_CACHE_EVICTION_SIZE;
 	int LOCATION_CACHE_EVICTION_SIZE_SIM;
+	double LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD;
+	double LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL;
 
 	int GET_RANGE_SHARD_LIMIT;
 	int WARM_RANGE_SHARD_LIMIT;
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index bd7b6f0e3a..038a26fbd7 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -198,6 +198,11 @@ struct ChangeFeedData : ReferenceCounted<ChangeFeedData> {
 	ChangeFeedData() : notAtLatest(1) {}
 };
 
+struct EndpointFailureInfo {
+	double startTime = 0;
+	double lastRefreshTime = 0;
+};
+
 class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAllocated<DatabaseContext>, NonCopyable {
 public:
 	static DatabaseContext* allocateOnForeignThread() {
@@ -241,6 +246,14 @@ public:
 	void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::False);
 	void invalidateCache(const KeyRangeRef&);
 
+	// Records that `endpoint` is failed on a healthy server.
+	void setFailedEndpointOnHealthyServer(const Endpoint& endpoint);
+
+	// Updates `endpoint` refresh time if the `endpoint` is a failed endpoint. If not, this does nothing.
+	void updateFailedEndpointRefreshTime(const Endpoint& endpoint);
+	Optional<EndpointFailureInfo> getEndpointFailureInfo(const Endpoint& endpoint);
+	void clearFailedEndpointOnHealthyServer(const Endpoint& endpoint);
+
 	bool sampleReadTags() const;
 	bool sampleOnCost(uint64_t cost) const;
 
@@ -394,6 +407,7 @@ public:
 	// Cache of location information
 	int locationCacheSize;
 	CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
+	std::unordered_map<Endpoint, EndpointFailureInfo> failedEndpointsOnHealthyServersInfo;
 
 	std::map<UID, StorageServerInfo*> server_interf;
 	std::map<UID, BlobWorkerInterface> blobWorker_interf; // blob workers don't change endpoints for the same ID
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 09d55c2c7f..e15ca52a94 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1598,6 +1598,32 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
 	locationCache.insert(KeyRangeRef(begin, end), Reference<LocationInfo>());
 }
 
+void DatabaseContext::setFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
+	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
+		failedEndpointsOnHealthyServersInfo[endpoint] = { /*startTime=*/now(), /*lastRefreshTime=*/now() };
+	}
+}
+
+void DatabaseContext::updateFailedEndpointRefreshTime(const Endpoint& endpoint) {
+	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
+		// The endpoint is not failed. Nothing to update.
+		return;
+	}
+
+	failedEndpointsOnHealthyServersInfo[endpoint].lastRefreshTime = now();
+}
+
+Optional<EndpointFailureInfo> DatabaseContext::getEndpointFailureInfo(const Endpoint& endpoint) {
+	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
+		return Optional<EndpointFailureInfo>();
+	}
+	return failedEndpointsOnHealthyServersInfo[endpoint];
+}
+
+void DatabaseContext::clearFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
+	failedEndpointsOnHealthyServersInfo.erase(endpoint);
+}
+
 Future<Void> DatabaseContext::onProxiesChanged() const {
 	return this->proxiesChangeTrigger.onTrigger();
 }
@@ -2465,14 +2491,42 @@ Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database con
 		return getKeyLocation_internal(cx, key, spanID, debugID, useProvisionalProxies, isBackward);
 	}
 
+	bool onlyEndpointFailedAndNeedRefresh = false;
 	for (int i = 0; i < ssi.second->size(); i++) {
-		if (IFailureMonitor::failureMonitor().onlyEndpointFailed(ssi.second->get(i, member).getEndpoint())) {
-			cx->invalidateCache(key);
-			ssi.second.clear();
-			return getKeyLocation_internal(cx, key, spanID, debugID, useProvisionalProxies, isBackward);
+		const Endpoint& endpoint = ssi.second->get(i, member).getEndpoint();
+		if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
+			// This endpoint is failed, but the server is still healthy. There are two cases this can happen:
+			//    - There is a recent bounce in the cluster where the endpoints in SSes get updated.
+			//    - The SS is failed and terminated on a server, but the server is kept running.
+			// To account for the first case, we invalidate the cache and issue GetKeyLocation requests to the proxy to
+			// update the cache with the new SS points. However, if the failure is caused by the second case, the
+			// requested key location will continue to be the failed endpoint until the data movement is finished. But
+			// every read will generate a GetKeyLocation request to the proxies (and still getting the failed endpoint
+			// back), which may overload the proxy and affect data movement speed. Therefore, we only refresh the
+			// location cache for short period of time, and after the initial grace period that we keep retrying
+			// resolving key location, we will slow it down to resolve it only once every
+			// `LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL`.
+			cx->setFailedEndpointOnHealthyServer(endpoint);
+			const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
+			ASSERT(failureInfo.present());
+			if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
+			    now() - failureInfo.get().lastRefreshTime >
+			        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
+				onlyEndpointFailedAndNeedRefresh = true;
+			}
+		} else {
+			cx->clearFailedEndpointOnHealthyServer(endpoint);
 		}
 	}
 
+	if (onlyEndpointFailedAndNeedRefresh) {
+		cx->invalidateCache(key);
+		for (int i = 0; i < ssi.second->size(); i++) {
+			cx->updateFailedEndpointRefreshTime(ssi.second->get(i, member).getEndpoint());
+		}
+		return getKeyLocation_internal(cx, key, spanID, debugID, useProvisionalProxies, isBackward);
+	}
+
 	return ssi;
 }
 
@@ -2555,21 +2609,37 @@ Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLoc
 
 	bool foundFailed = false;
 	for (const auto& [range, locInfo] : locations) {
-		bool onlyEndpointFailed = false;
+		bool onlyEndpointFailedAndNeedRefresh = false;
 		for (int i = 0; i < locInfo->size(); i++) {
-			if (IFailureMonitor::failureMonitor().onlyEndpointFailed(locInfo->get(i, member).getEndpoint())) {
-				onlyEndpointFailed = true;
-				break;
+			const Endpoint& endpoint = locInfo->get(i, member).getEndpoint();
+			if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
+				// Please refer to `getKeyLocation` about why we keep track of endpoint failure starting and refreshing
+				// time.
+				cx->setFailedEndpointOnHealthyServer(endpoint);
+				const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
+				ASSERT(failureInfo.present());
+				if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
+				    now() - failureInfo.get().lastRefreshTime >
+				        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
+					onlyEndpointFailedAndNeedRefresh = true;
+				}
+			} else {
+				cx->clearFailedEndpointOnHealthyServer(endpoint);
 			}
 		}
 
-		if (onlyEndpointFailed) {
+		if (onlyEndpointFailedAndNeedRefresh) {
 			cx->invalidateCache(range.begin);
 			foundFailed = true;
 		}
 	}
 
 	if (foundFailed) {
+		for (const auto& [range, locInfo] : locations) {
+			for (int i = 0; i < locInfo->size(); i++) {
+				cx->updateFailedEndpointRefreshTime(locInfo->get(i, member).getEndpoint());
+			}
+		}
 		return getKeyRangeLocations_internal(cx, keys, limit, reverse, spanID, debugID, useProvisionalProxies);
 	}
 

From 0bdae35a2ca14f01f19c6c4341fdf4eff26ec871 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 24 Feb 2022 09:20:15 -0800
Subject: [PATCH 113/210] Address comments and move
 updateFailedEndpointRefreshTime

---
 fdbclient/NativeAPI.actor.cpp | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index e15ca52a94..d4abc5cf8a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2513,6 +2513,7 @@ Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database con
 			    now() - failureInfo.get().lastRefreshTime >
 			        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
 				onlyEndpointFailedAndNeedRefresh = true;
+				cx->updateFailedEndpointRefreshTime(ssi.second->get(i, member).getEndpoint());
 			}
 		} else {
 			cx->clearFailedEndpointOnHealthyServer(endpoint);
@@ -2521,9 +2522,8 @@ Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database con
 
 	if (onlyEndpointFailedAndNeedRefresh) {
 		cx->invalidateCache(key);
-		for (int i = 0; i < ssi.second->size(); i++) {
-			cx->updateFailedEndpointRefreshTime(ssi.second->get(i, member).getEndpoint());
-		}
+
+		// Refresh the cache with a new getKeyLocations made to proxies.
 		return getKeyLocation_internal(cx, key, spanID, debugID, useProvisionalProxies, isBackward);
 	}
 
@@ -2622,6 +2622,7 @@ Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLoc
 				    now() - failureInfo.get().lastRefreshTime >
 				        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
 					onlyEndpointFailedAndNeedRefresh = true;
+					cx->updateFailedEndpointRefreshTime(locInfo->get(i, member).getEndpoint());
 				}
 			} else {
 				cx->clearFailedEndpointOnHealthyServer(endpoint);
@@ -2635,11 +2636,7 @@ Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLoc
 	}
 
 	if (foundFailed) {
-		for (const auto& [range, locInfo] : locations) {
-			for (int i = 0; i < locInfo->size(); i++) {
-				cx->updateFailedEndpointRefreshTime(locInfo->get(i, member).getEndpoint());
-			}
-		}
+		// Refresh the cache with a new getKeyRangeLocations made to proxies.
 		return getKeyRangeLocations_internal(cx, keys, limit, reverse, spanID, debugID, useProvisionalProxies);
 	}
 

From 3a3b44efe27ca4ae1600964b24b8f56c19fa5ae2 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Thu, 24 Feb 2022 19:13:24 -0800
Subject: [PATCH 114/210] Fix macOS CI broken by #6449.

foundationdb/fdbserver/CoroFlow.actor.cpp:262:8: error: no member named 'errorUnsuppressed' in 'BaseTraceEvent'
---
 fdbserver/CoroFlow.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/CoroFlow.actor.cpp b/fdbserver/CoroFlow.actor.cpp
index 948e1f8fd1..23f08b66cf 100644
--- a/fdbserver/CoroFlow.actor.cpp
+++ b/fdbserver/CoroFlow.actor.cpp
@@ -256,10 +256,10 @@ public:
 
 		pool->queueLock.enter();
 		TraceEvent("WorkPool_Stop")
+		    .errorUnsuppressed(e)
 		    .detail("Workers", pool->workers.size())
 		    .detail("Idle", pool->idle.size())
-		    .detail("Work", pool->work.size())
-		    .errorUnsuppressed(e);
+		    .detail("Work", pool->work.size());
 
 		for (uint32_t i = 0; i < pool->work.size(); i++)
 			pool->work[i]->cancel(); // What if cancel() does something to this?

From 3e761aef3ba5a901227eaea20090805edb30a4c8 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Thu, 24 Feb 2022 15:32:19 -0800
Subject: [PATCH 115/210] Add resetConnectionString() function to
 ClusterConnectionString.

---
 fdbclient/CoordinationInterface.h |  3 +++
 fdbclient/MonitorLeader.actor.cpp | 20 ++++++--------------
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index 9873817a88..1fd7a5f0ca 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -93,6 +93,9 @@ public:
 	// This one should only be used when resolving asynchronously is impossible. For all other cases, resolveHostnames()
 	// should be preferred.
 	void resolveHostnamesBlocking();
+	// This function derives the member connectionString from the current key, coordinators and hostnames.
+	void resetConnectionString();
+
 	void resetToUnresolved();
 
 	ConnectionStringStatus status = RESOLVED;
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 4ba785cd42..e63824466f 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -178,6 +178,10 @@ void ClusterConnectionString::resetToUnresolved() {
 	}
 }
 
+void ClusterConnectionString::resetConnectionString() {
+	connectionString = toString();
+}
+
 void ClusterConnectionString::parseConnString() {
 	// Split on '@' into key@addrs
 	int pAt = connectionString.find_first_of('@');
@@ -444,26 +448,14 @@ ClusterConnectionString::ClusterConnectionString(const std::vector<NetworkAddres
   : status(RESOLVED), coords(servers) {
 	std::string keyString = key.toString();
 	parseKey(keyString);
-	connectionString = keyString + "@";
-	for (int i = 0; i < coords.size(); i++) {
-		if (i) {
-			connectionString += ',';
-		}
-		connectionString += coords[i].toString();
-	}
+	resetConnectionString();
 }
 
 ClusterConnectionString::ClusterConnectionString(const std::vector<Hostname>& hosts, Key key)
   : status(UNRESOLVED), hostnames(hosts) {
 	std::string keyString = key.toString();
 	parseKey(keyString);
-	connectionString = keyString + "@";
-	for (int i = 0; i < hostnames.size(); i++) {
-		if (i) {
-			connectionString += ',';
-		}
-		connectionString += hostnames[i].toString();
-	}
+	resetConnectionString();
 }
 
 void ClusterConnectionString::parseKey(const std::string& key) {

From 70d723f70ce56b5c4087666815cd2fb1f5f70997 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Thu, 24 Feb 2022 16:14:06 -0800
Subject: [PATCH 116/210] Move the construction of ConnectionString in
 changeQuorumChecker() to coordinatorsCommitActor().

---
 fdbclient/CoordinationInterface.h   |  2 +-
 fdbclient/ManagementAPI.actor.cpp   | 29 +++++++++++++++--------------
 fdbclient/ManagementAPI.actor.h     |  2 +-
 fdbclient/SpecialKeySpace.actor.cpp | 21 ++++++++++++---------
 4 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index 1fd7a5f0ca..bd37fb3f55 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -97,6 +97,7 @@ public:
 	void resetConnectionString();
 
 	void resetToUnresolved();
+	void parseKey(const std::string& key);
 
 	ConnectionStringStatus status = RESOLVED;
 	AsyncTrigger resolveFinish;
@@ -106,7 +107,6 @@ public:
 
 private:
 	void parseConnString();
-	void parseKey(const std::string& key);
 	Key key, keyDesc;
 	std::string connectionString;
 };
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 0239cd23af..13299352f5 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -772,7 +772,7 @@ ACTOR Future<std::vector<NetworkAddress>> getCoordinators(Database cx) {
 
 ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
                                                                Reference<IQuorumChange> change,
-                                                               std::vector<NetworkAddress>* desiredCoordinators) {
+                                                               ClusterConnectionString* conn) {
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
@@ -789,38 +789,39 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
 
 	state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
-	if (!desiredCoordinators->size()) {
-		std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
+	if (!conn->coords.size()) {
+		std::vector<NetworkAddress> desiredCoordinatorAddresses = wait(change->getDesiredCoordinators(
 		    tr,
 		    old.coordinators(),
 		    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
 		    result));
-		*desiredCoordinators = _desiredCoordinators;
+		conn->coords = desiredCoordinatorAddresses;
 	}
 
 	if (result != CoordinatorsResult::SUCCESS)
 		return result;
 
-	if (!desiredCoordinators->size())
+	if (!conn->coordinators().size())
 		return CoordinatorsResult::INVALID_NETWORK_ADDRESSES;
 
-	std::sort(desiredCoordinators->begin(), desiredCoordinators->end());
+	std::sort(conn->coords.begin(), conn->coords.end());
 
 	std::string newName = change->getDesiredClusterKeyName();
 	if (newName.empty())
 		newName = old.clusterKeyName().toString();
 
-	if (old.coordinators() == *desiredCoordinators && old.clusterKeyName() == newName)
+	if (old.coordinators() == conn->coordinators() && old.clusterKeyName() == newName)
 		return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
 
-	state ClusterConnectionString conn(*desiredCoordinators,
-	                                   StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
+	std::string key(newName + ':' + deterministicRandom()->randomAlphaNumeric(32));
+	conn->parseKey(key);
+	conn->resetConnectionString();
 
 	if (g_network->isSimulated()) {
 		int i = 0;
 		int protectedCount = 0;
-		while ((protectedCount < ((desiredCoordinators->size() / 2) + 1)) && (i < desiredCoordinators->size())) {
-			auto process = g_simulator.getProcessByAddress((*desiredCoordinators)[i]);
+		while ((protectedCount < ((conn->coordinators().size() / 2) + 1)) && (i < conn->coordinators().size())) {
+			auto process = g_simulator.getProcessByAddress(conn->coordinators()[i]);
 			auto addresses = process->addresses;
 
 			if (!process->isReliable()) {
@@ -832,14 +833,14 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 			if (addresses.secondaryAddress.present()) {
 				g_simulator.protectedAddresses.insert(process->addresses.secondaryAddress.get());
 			}
-			TraceEvent("ProtectCoordinator").detail("Address", (*desiredCoordinators)[i]).backtrace();
+			TraceEvent("ProtectCoordinator").detail("Address", conn->coordinators()[i]).backtrace();
 			protectedCount++;
 			i++;
 		}
 	}
 
 	std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-	ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(conn)));
+	ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(*conn)));
 
 	leaderServers.reserve(coord.clientLeaderServers.size());
 	for (int i = 0; i < coord.clientLeaderServers.size(); i++)
@@ -851,7 +852,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		when(wait(waitForAll(leaderServers))) {}
 		when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
 	}
-	tr->set(coordinatorsKey, conn.toString());
+	tr->set(coordinatorsKey, conn->toString());
 	return Optional<CoordinatorsResult>();
 }
 
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index d07a743a5e..a2adbb7025 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -56,7 +56,7 @@ struct IQuorumChange : ReferenceCounted<IQuorumChange> {
 // Change to use the given set of coordination servers
 ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
                                                                Reference<IQuorumChange> change,
-                                                               std::vector<NetworkAddress>* desiredCoordinators);
+                                                               ClusterConnectionString* conn);
 ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
 Reference<IQuorumChange> autoQuorumChange(int desired = -1);
 Reference<IQuorumChange> noQuorumChange();
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 6c1db7639f..6418e1d1e7 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -1628,7 +1628,8 @@ Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw, K
 
 ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	state Reference<IQuorumChange> change;
-	state std::vector<NetworkAddress> addressesVec;
+	state ClusterConnectionString
+	    conn; // We don't care about the Key here, it will be overrode in changeQuorumChecker().
 	state std::vector<std::string> process_address_strs;
 	state Optional<std::string> msg;
 	state int index;
@@ -1650,11 +1651,12 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 		}
 		for (index = 0; index < process_address_strs.size(); index++) {
 			try {
-				auto a = NetworkAddress::parse(process_address_strs[index]);
-				if (!a.isValid())
+				NetworkAddress a = NetworkAddress::parse(process_address_strs[index]);
+				if (!a.isValid()) {
 					parse_error = true;
-				else
-					addressesVec.push_back(a);
+				} else {
+					conn.coords.push_back(a);
+				}
 			} catch (Error& e) {
 				TraceEvent(SevDebug, "SpecialKeysNetworkParseError").error(e);
 				parse_error = true;
@@ -1670,8 +1672,9 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 		}
 	}
 
-	if (addressesVec.size())
-		change = specifiedQuorumChange(addressesVec);
+	wait(conn.resolveHostnames());
+	if (conn.coordinators().size())
+		change = specifiedQuorumChange(conn.coordinators());
 	else
 		change = noQuorumChange();
 
@@ -1693,10 +1696,10 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	ASSERT(change.isValid());
 
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsStart")
-	    .detail("NewAddresses", describe(addressesVec))
+	    .detail("NewAddresses", describe(conn.coordinators()))
 	    .detail("Description", entry.first ? entry.second.get().toString() : "");
 
-	Optional<CoordinatorsResult> r = wait(changeQuorumChecker(&ryw->getTransaction(), change, &addressesVec));
+	Optional<CoordinatorsResult> r = wait(changeQuorumChecker(&ryw->getTransaction(), change, &conn));
 
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsFinish")
 	    .detail("Result", r.present() ? static_cast<int>(r.get()) : -1); // -1 means success

From 06b1d06d38c58d7049048753bac92cc3523c22cd Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Thu, 24 Feb 2022 16:34:30 -0800
Subject: [PATCH 117/210] Support hostname in coordinators commands.

---
 fdbcli/CoordinatorsCommand.actor.cpp  | 39 ++++++++++++++++++---------
 fdbclient/ManagementAPI.actor.cpp     |  2 ++
 fdbclient/SpecialKeySpace.actor.cpp   | 30 ++++++++++++---------
 fdbclient/StatusClient.actor.cpp      |  1 +
 fdbserver/ClusterController.actor.cpp |  1 +
 fdbserver/ClusterRecovery.actor.cpp   |  4 ++-
 flow/network.cpp                      |  2 +-
 flow/network.h                        | 19 ++++++++++---
 8 files changed, 69 insertions(+), 29 deletions(-)

diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.actor.cpp
index 261090c5ed..f274f17df7 100644
--- a/fdbcli/CoordinatorsCommand.actor.cpp
+++ b/fdbcli/CoordinatorsCommand.actor.cpp
@@ -65,13 +65,14 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 	state StringRef new_cluster_description;
 	state std::string auto_coordinators_str;
 	StringRef nameTokenBegin = LiteralStringRef("description=");
-	for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok)
+	for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) {
 		if (tok->startsWith(nameTokenBegin)) {
 			new_cluster_description = tok->substr(nameTokenBegin.size());
 			std::copy(tok + 1, tokens.end(), tok);
 			tokens.resize(tokens.size() - 1);
 			break;
 		}
+	}
 
 	state bool automatic = tokens.size() == 2 && tokens[1] == LiteralStringRef("auto");
 	state Reference<ITransaction> tr = db->createTransaction();
@@ -96,18 +97,32 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 				tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str);
 			} else if (tokens.size() > 1) {
 				state std::set<NetworkAddress> new_coordinators_addresses;
-				state std::vector<std::string> newAddresslist;
+				state std::set<Hostname> new_coordinators_hostnames;
+				state std::vector<std::string> newCoordinatorslist;
 				state std::vector<StringRef>::iterator t;
 				for (t = tokens.begin() + 1; t != tokens.end(); ++t) {
 					try {
-						// TODO(renxuan): add hostname parsing here.
-						auto const& addr = NetworkAddress::parse(t->toString());
-						if (new_coordinators_addresses.count(addr)) {
-							fprintf(stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
-							return true;
+						if (Hostname::isHostname(t->toString())) {
+							// We do not resolve hostnames here. We commit them as is.
+							const auto& hostname = Hostname::parse(t->toString());
+							if (new_coordinators_hostnames.count(hostname)) {
+								fprintf(stderr,
+								        "ERROR: passed redundant coordinators: `%s'\n",
+								        hostname.toString().c_str());
+								return true;
+							}
+							new_coordinators_hostnames.insert(hostname);
+							newCoordinatorslist.push_back(hostname.toString());
+						} else {
+							const auto& addr = NetworkAddress::parse(t->toString());
+							if (new_coordinators_addresses.count(addr)) {
+								fprintf(
+								    stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
+								return true;
+							}
+							new_coordinators_addresses.insert(addr);
+							newCoordinatorslist.push_back(addr.toString());
 						}
-						new_coordinators_addresses.insert(addr);
-						newAddresslist.push_back(addr.toString());
 					} catch (Error& e) {
 						if (e.code() == error_code_connection_string_invalid) {
 							fprintf(
@@ -117,12 +132,12 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 						throw;
 					}
 				}
-				std::string new_addresses_str = boost::algorithm::join(newAddresslist, ", ");
-				tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_addresses_str);
+				std::string new_coordinators_str = boost::algorithm::join(newCoordinatorslist, ", ");
+				tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_coordinators_str);
 			}
 			wait(safeThreadFutureToFuture(tr->commit()));
 			// commit should always fail here
-			// if coordinators are changed, we should get commit_unknown() error
+			// If the commit succeeds, the coordinators change and the commit will fail with commit_unknown_result().
 			ASSERT(false);
 		} catch (Error& e) {
 			state Error err(e);
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 13299352f5..99ebea516d 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -783,6 +783,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
 
 	state ClusterConnectionString old(currentKey.get().toString());
+	wait(old.resolveHostnames());
 	if (tr->getDatabase()->getConnectionRecord() &&
 	    old.clusterKeyName().toString() !=
 	        tr->getDatabase()->getConnectionRecord()->getConnectionString().clusterKeyName())
@@ -805,6 +806,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		return CoordinatorsResult::INVALID_NETWORK_ADDRESSES;
 
 	std::sort(conn->coords.begin(), conn->coords.end());
+	std::sort(conn->hostnames.begin(), conn->hostnames.end());
 
 	std::string newName = change->getDesiredClusterKeyName();
 	if (newName.empty())
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 6418e1d1e7..af4fe3e470 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -1630,7 +1630,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	state Reference<IQuorumChange> change;
 	state ClusterConnectionString
 	    conn; // We don't care about the Key here, it will be overrode in changeQuorumChecker().
-	state std::vector<std::string> process_address_strs;
+	state std::vector<std::string> process_address_or_hostname_strs;
 	state Optional<std::string> msg;
 	state int index;
 	state bool parse_error = false;
@@ -1641,21 +1641,26 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	if (processes_entry.first) {
 		ASSERT(processes_entry.second.present()); // no clear should be seen here
 		auto processesStr = processes_entry.second.get().toString();
-		boost::split(process_address_strs, processesStr, [](char c) { return c == ','; });
-		if (!process_address_strs.size()) {
+		boost::split(process_address_or_hostname_strs, processesStr, [](char c) { return c == ','; });
+		if (!process_address_or_hostname_strs.size()) {
 			return ManagementAPIError::toJsonString(
 			    false,
 			    "coordinators",
 			    "New coordinators\' processes are empty, please specify new processes\' network addresses with format "
-			    "\"IP:PORT,IP:PORT,...,IP:PORT\"");
+			    "\"IP:PORT,IP:PORT,...,IP:PORT\" or \"HOSTNAME:PORT,HOSTNAME:PORT,...,HOSTNAME:PORT\"");
 		}
-		for (index = 0; index < process_address_strs.size(); index++) {
+		for (index = 0; index < process_address_or_hostname_strs.size(); index++) {
 			try {
-				NetworkAddress a = NetworkAddress::parse(process_address_strs[index]);
-				if (!a.isValid()) {
-					parse_error = true;
+				if (Hostname::isHostname(process_address_or_hostname_strs[index])) {
+					conn.hostnames.push_back(Hostname::parse(process_address_or_hostname_strs[index]));
+					conn.status = ClusterConnectionString::ConnectionStringStatus::UNRESOLVED;
 				} else {
-					conn.coords.push_back(a);
+					NetworkAddress a = NetworkAddress::parse(process_address_or_hostname_strs[index]);
+					if (!a.isValid()) {
+						parse_error = true;
+					} else {
+						conn.coords.push_back(a);
+					}
 				}
 			} catch (Error& e) {
 				TraceEvent(SevDebug, "SpecialKeysNetworkParseError").error(e);
@@ -1663,9 +1668,9 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 			}
 
 			if (parse_error) {
-				std::string error =
-				    "ERROR: \'" + process_address_strs[index] + "\' is not a valid network endpoint address\n";
-				if (process_address_strs[index].find(":tls") != std::string::npos)
+				std::string error = "ERROR: \'" + process_address_or_hostname_strs[index] +
+				                    "\' is not a valid network endpoint address\n";
+				if (process_address_or_hostname_strs[index].find(":tls") != std::string::npos)
 					error += "        Do not include the `:tls' suffix when naming a process\n";
 				return ManagementAPIError::toJsonString(false, "coordinators", error);
 			}
@@ -1696,6 +1701,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	ASSERT(change.isValid());
 
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsStart")
+	    .detail("NewHostnames", conn.hostnames.size() ? describe(conn.hostnames) : "N/A")
 	    .detail("NewAddresses", describe(conn.coordinators()))
 	    .detail("Description", entry.first ? entry.second.get().toString() : "");
 
diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp
index 2829771ba8..fc7a77b17a 100644
--- a/fdbclient/StatusClient.actor.cpp
+++ b/fdbclient/StatusClient.actor.cpp
@@ -306,6 +306,7 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<I
                                                                      bool* quorum_reachable,
                                                                      int* coordinatorsFaultTolerance) {
 	try {
+		wait(connRecord->resolveHostnames());
 		state ClientCoordinators coord(connRecord);
 		state StatusObject statusObj;
 
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 256371852b..5860e9bd84 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2599,6 +2599,7 @@ ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> connRec
 	state bool hasConnected = false;
 	loop {
 		try {
+			wait(connRecord->resolveHostnames());
 			ServerCoordinators coordinators(connRecord);
 			wait(clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, configDBType));
 		} catch (Error& e) {
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index 41042771b0..512539a607 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -673,7 +673,9 @@ ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
 		}
 
 		try {
-			wait(self->cstate.move(ClusterConnectionString(changeCoordinatorsRequest.newConnectionString.toString())));
+			state ClusterConnectionString conn(changeCoordinatorsRequest.newConnectionString.toString());
+			wait(conn.resolveHostnames());
+			wait(self->cstate.move(conn));
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled)
 				changeCoordinatorsRequest.reply.sendError(e);
diff --git a/flow/network.cpp b/flow/network.cpp
index 515d425eae..6dd84869fa 100644
--- a/flow/network.cpp
+++ b/flow/network.cpp
@@ -63,7 +63,7 @@ bool IPAddress::isValid() const {
 	return std::get<uint32_t>(addr) != 0;
 }
 
-Hostname Hostname::parse(std::string const& s) {
+Hostname Hostname::parse(const std::string& s) {
 	if (s.empty()) {
 		throw connection_string_invalid();
 	}
diff --git a/flow/network.h b/flow/network.h
index a609ba50d3..6ab9e655ed 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -143,18 +143,31 @@ struct Hostname {
 	Hostname(std::string host, std::string service, bool isTLS) : host(host), service(service), isTLS(isTLS) {}
 	Hostname() : host(""), service(""), isTLS(false) {}
 
+	bool operator==(const Hostname& r) const { return host == r.host && service == r.service && isTLS == r.isTLS; }
+	bool operator!=(const Hostname& r) const { return !(*this == r); }
+	bool operator<(const Hostname& r) const {
+		if (isTLS != r.isTLS)
+			return isTLS < r.isTLS;
+		else if (host != r.host)
+			return host < r.host;
+		return service < r.service;
+	}
+	bool operator>(const Hostname& r) const { return r < *this; }
+	bool operator<=(const Hostname& r) const { return !(*this > r); }
+	bool operator>=(const Hostname& r) const { return !(*this < r); }
+
 	// Allow hostnames in forms like following:
 	//    hostname:1234
 	//    host.name:1234
 	//    host-name:1234
 	//    host-name_part1.host-name_part2:1234:tls
-	static bool isHostname(std::string& s) {
+	static bool isHostname(const std::string& s) {
 		std::regex validation("^([\\w\\-]+\\.?)+:([\\d]+){1,}(:tls)?$");
 		std::regex ipv4Validation("^([\\d]{1,3}\\.?){4,}:([\\d]+){1,}(:tls)?$");
 		return !std::regex_match(s, ipv4Validation) && std::regex_match(s, validation);
 	}
 
-	static Hostname parse(std::string const& str);
+	static Hostname parse(const std::string& s);
 
 	std::string toString() const { return host + ":" + service + (isTLS ? ":tls" : ""); }
 };
@@ -262,7 +275,7 @@ struct NetworkAddress {
 	  : NetworkAddress(ip, port, false, false, NetworkAddressFromHostname::False) {}
 
 	bool operator==(NetworkAddress const& r) const { return ip == r.ip && port == r.port && flags == r.flags; }
-	bool operator!=(NetworkAddress const& r) const { return ip != r.ip || port != r.port || flags != r.flags; }
+	bool operator!=(NetworkAddress const& r) const { return !(*this == r); }
 	bool operator<(NetworkAddress const& r) const {
 		if (flags != r.flags)
 			return flags < r.flags;

From 5b6098b5beb6a7f92528524e55b49f6f2bdfb6bb Mon Sep 17 00:00:00 2001
From: Mohamed Oulmahdi <oulmahdi@doxense.com>
Date: Fri, 25 Feb 2022 16:25:45 +0100
Subject: [PATCH 118/210] Fix Windows build broken by #6449

---
 flow/Platform.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 084eaa5039..d42c8e721c 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -622,7 +622,7 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) {
 	ULARGE_INTEGER totalFreeSpace;
 	if (!GetDiskFreeSpaceEx(fullPath.c_str(), &freeSpace, &totalSpace, &totalFreeSpace)) {
 		Error e = systemErrorCodeToError();
-		TraceEvent(SevError, "DiskFreeError").detail("Path", fullPath).GetLastError().error(e);
+		TraceEvent(SevError, "DiskFreeError").error(e).detail("Path", fullPath).GetLastError();
 		throw e;
 	}
 	total = std::min((uint64_t)std::numeric_limits<int64_t>::max(), totalSpace.QuadPart);
@@ -2353,7 +2353,7 @@ bool createDirectory(std::string const& directory) {
 		}
 	}
 	Error e = systemErrorCodeToError();
-	TraceEvent(SevError, "CreateDirectory").detail("Directory", directory).GetLastError().error(e);
+	TraceEvent(SevError, "CreateDirectory").error(e).detail("Directory", directory).GetLastError();
 	throw e;
 #elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
 	size_t sep = 0;
@@ -2500,7 +2500,7 @@ std::string abspath(std::string const& path, bool resolveLinks, bool mustExist)
 	if (!GetFullPathName(path.c_str(), MAX_PATH, nameBuffer, nullptr) || (mustExist && !fileExists(nameBuffer))) {
 		Error e = systemErrorCodeToError();
 		Severity sev = e.code() == error_code_io_error ? SevError : SevWarnAlways;
-		TraceEvent(sev, "AbsolutePathError").detail("Path", path).GetLastError().error(e);
+		TraceEvent(sev, "AbsolutePathError").error(e).detail("Path", path).GetLastError();
 		throw e;
 	}
 	// Not totally obvious from the help whether GetFullPathName canonicalizes slashes, so let's do it...

From 57e931b489fd841810afeb30ff3f320b9897846f Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 24 Feb 2022 21:36:14 -0800
Subject: [PATCH 119/210] Update 6.3.24 release notes

---
 .../sphinx/source/release-notes/release-notes-630.rst     | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst
index 9deaf9ec27..0876faa1fc 100644
--- a/documentation/sphinx/source/release-notes/release-notes-630.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-630.rst
@@ -2,8 +2,16 @@
 Release Notes
 #############
 
+6.3.24
+======
+* Fixed a bug where get key location can overload proxies. `(PR #6453) <https://github.com/apple/foundationdb/pull/6453>`_ 
+* Added a mechanism that can reduce the number of empty peek reply by not always returning empty peek reply immediately. `(PR #6413) <https://github.com/apple/foundationdb/pull/6413>`_
+* Enable TLS support for Windows. `(PR #6193) <https://github.com/apple/foundationdb/pull/6193>`_
+* Fixed a bug where a shard gets merged too soon. `(PR #6115) <https://github.com/apple/foundationdb/pull/6115>`_
+
 6.3.23
 ======
+* Add AWS v4 header support for backup. `(PR #6025) <https://github.com/apple/foundationdb/pull/6025>`_
 * Fixed a bug that remoteDCIsHealthy logic is not guarded by CC_ENABLE_WORKER_HEALTH_MONITOR, which may prevent HA failback. `(PR #6106) <https://github.com/apple/foundationdb/pull/6106>`_
 * Fixed a race condition with updating the coordinated state and updating the master registration. `(PR #6088) <https://github.com/apple/foundationdb/pull/6088>`_
 * Changed dbinfo broadcast to be explicitly requested by the worker registration message. `(PR #6073) <https://github.com/apple/foundationdb/pull/6073>`_

From 7a9217a392fdb8568b10aa430c8c55b59756b5ac Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Fri, 25 Feb 2022 12:54:31 -0800
Subject: [PATCH 120/210] Add contrib/debug_determinism (#6389)

* Add contrib/debug_determinism

Add an instrumentation-based technique for debugging unseen mismatches. Also guard a few existing sources of nondeterminism that don't affect unseen with the DEBUG_DETERMINISM macro.

Also change the simulated run loop to not run as the only task inside the real run loop, since that was a source of nondeterminism.

Also fix nondeterminism from calling timer_int

* Add StorageMetadataType::currentTime

Basically a deterministic-in-simulation version of timer_int that we can
use instead of timer_int for StorageMetadataType::createdTime
---
 contrib/CMakeLists.txt                        |  1 +
 contrib/debug_determinism/CMakeLists.txt      |  5 ++
 contrib/debug_determinism/README.md           | 45 ++++++++++++++++
 .../debug_determinism/debug_determinism.cpp   | 52 +++++++++++++++++++
 fdbclient/FDBTypes.h                          |  4 +-
 fdbrpc/sim2.actor.cpp                         | 13 ++---
 fdbserver/DDTeamCollection.actor.cpp          |  2 +-
 fdbserver/MoveKeys.actor.cpp                  |  4 +-
 flow/Arena.cpp                                |  2 +
 flow/FastAlloc.cpp                            |  2 +
 flow/SystemMonitor.cpp                        |  2 +
 flow/flow.cpp                                 |  2 +-
 12 files changed, 119 insertions(+), 15 deletions(-)
 create mode 100644 contrib/debug_determinism/CMakeLists.txt
 create mode 100644 contrib/debug_determinism/README.md
 create mode 100644 contrib/debug_determinism/debug_determinism.cpp

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 63ed617212..47705943be 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(fmt-8.0.1)
 if(NOT WIN32)
+  add_subdirectory(debug_determinism)
   add_subdirectory(monitoring)
   add_subdirectory(TraceLogHelper)
   add_subdirectory(TestHarness)
diff --git a/contrib/debug_determinism/CMakeLists.txt b/contrib/debug_determinism/CMakeLists.txt
new file mode 100644
index 0000000000..0aefcfc59f
--- /dev/null
+++ b/contrib/debug_determinism/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_library(debug_determinism STATIC debug_determinism.cpp)
+
+# So that we can link to libfdb_c.so. Not strictly necessary but convenient for use with our
+# TRACE_PC_GUARD_INSTRUMENTATION_LIB cmake option
+target_compile_options(debug_determinism PRIVATE -fPIC)
\ No newline at end of file
diff --git a/contrib/debug_determinism/README.md b/contrib/debug_determinism/README.md
new file mode 100644
index 0000000000..3b327394dd
--- /dev/null
+++ b/contrib/debug_determinism/README.md
@@ -0,0 +1,45 @@
+Utilities for debugging unseed mismatches for foundationdb simulation tests.
+
+99/100 times the source of the nondeterminism is use of uninitialized memory and
+what you want to do is build with `-DUSE_VALGRIND=ON` and run simulations under
+valgrind.
+
+Common sources of nondeterminism and specialized tools to find them.
+1. Use of uninitialized memory (use valgrind!)
+1. Memory errors (use valgrind and/or asan)
+1. Undefined behavior (use ubsan. You can also try _GLIBCXX_DEBUG)
+
+If it's not any of these then now it's time to try this technique. Look for
+
+1. Call to some kind of "get current time" function that's not in `INetwork`
+1. Depending on the relative ordering of allocated memory. E.g. Using heap-allocated pointers as keys in a `std::map`.
+1. Inspecting something about the current state of the system (e.g. free disk space)
+1. Depending on iteration order of an unordered map
+
+# Quickstart
+
+Set these cmake flags
+
+```
+-DTRACE_PC_GUARD_INSTRUMENTATION_LIB=$BUILDDIR/lib/libdebug_determinism.a
+```
+
+and change `#define DEBUG_DETERMINISM 0` to `#define DEBUG_DETERMINISM 1` in
+flow/Platform.h. This disables several known sources of nondeterminism that
+don't affect unseeds.
+
+For reasons I don't fully understand, it appears that sqlite exhibits some
+nondeterminism if you don't add `#define SQLITE_OMIT_LOOKASIDE` to the top of
+fdbserver/sqlite/sqlite3.amalgamation.c, so you probably want to do that too.
+
+Now when you run an fdbserver simulation, it will write a file `out.bin` in the
+current directory which contains the sequence of edges in the control flow graph
+that were encountered during the simulation. If you rename `out.bin` to `in.bin`
+and then re-run, the simulation will validate that the sequence of edges is the
+same as the last run. If it's not, then the simulation will enter an infinite
+loop at the first difference and print a message. Then you probably want to
+attach gdb to the process and investigate from there.
+
+You'll need to make sure you delete the `simfdb` folder before each run, because
+otherwise you'll take a different codepath for deleting the `simfdb` folder at
+the beginning of simulation.
diff --git a/contrib/debug_determinism/debug_determinism.cpp b/contrib/debug_determinism/debug_determinism.cpp
new file mode 100644
index 0000000000..02982a8cbb
--- /dev/null
+++ b/contrib/debug_determinism/debug_determinism.cpp
@@ -0,0 +1,52 @@
+#include <stdint.h>
+#include <stdio.h>
+
+namespace {
+FILE* out = nullptr;
+FILE* in = nullptr;
+void loop_forever() {
+	// Try to convince the optimizer not to optimize away this loop
+	static volatile uint64_t x = 0;
+	for (;;) {
+		++x;
+	}
+}
+} // namespace
+
+// This callback is inserted by the compiler as a module constructor
+// into every DSO. 'start' and 'stop' correspond to the
+// beginning and end of the section with the guards for the entire
+// binary (executable or DSO). The callback will be called at least
+// once per DSO and may be called multiple times with the same parameters.
+extern "C" void __sanitizer_cov_trace_pc_guard_init(uint32_t* start, uint32_t* stop) {
+	in = fopen("in.bin", "r");
+	out = fopen("out.bin", "w");
+	static uint64_t N; // Counter for the guards.
+	if (start == stop || *start)
+		return; // Initialize only once.
+	for (uint32_t* x = start; x < stop; x++) {
+		*x = ++N; // Guards should start from 1.
+	}
+}
+
+// This callback is inserted by the compiler on every edge in the
+// control flow (some optimizations apply).
+// Typically, the compiler will emit the code like this:
+//    if(*guard)
+//      __sanitizer_cov_trace_pc_guard(guard);
+// But for large functions it will emit a simple call:
+//    __sanitizer_cov_trace_pc_guard(guard);
+extern "C" void __sanitizer_cov_trace_pc_guard(uint32_t* guard) {
+	if (!guard) {
+		return;
+	}
+	fwrite(guard, 1, sizeof(*guard), out);
+	if (in) {
+		uint32_t theirs;
+		fread(&theirs, 1, sizeof(theirs), in);
+		if (*guard != theirs) {
+			printf("Non-determinism detected\n");
+			loop_forever();
+		}
+	}
+}
\ No newline at end of file
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 8351d86ccb..8a9bfe6ac0 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1204,10 +1204,12 @@ struct ReadBlobGranuleContext {
 struct StorageMetadataType {
 	constexpr static FileIdentifier file_identifier = 732123;
 	// when the SS is initialized
-	uint64_t createdTime; // comes from Platform::timer_int()
+	uint64_t createdTime; // comes from currentTime()
 	StorageMetadataType() : createdTime(0) {}
 	StorageMetadataType(uint64_t t) : createdTime(t) {}
 
+	static uint64_t currentTime() { return g_network->timer() * 1e9; }
+
 	// To change this serialization, ProtocolVersion::StorageMetadata must be updated, and downgrades need
 	// to be considered
 	template <class Ar>
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index b83f1d81b0..d68652421f 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -1123,11 +1123,9 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> runLoop(Sim2* self) {
-		state ISimulator::ProcessInfo* callingMachine = self->currentProcess;
+	static void runLoop(Sim2* self) {
+		ISimulator::ProcessInfo* callingMachine = self->currentProcess;
 		while (!self->isStopped) {
-			wait(self->net2->yield(TaskPriority::DefaultYield));
-
 			self->mutex.enter();
 			if (self->tasks.size() == 0) {
 				self->mutex.leave();
@@ -1144,18 +1142,13 @@ public:
 			self->yielded = false;
 		}
 		self->currentProcess = callingMachine;
-		self->net2->stop();
 		for (auto& fn : self->stopCallbacks) {
 			fn();
 		}
-		return Void();
 	}
 
 	// Implement ISimulator interface
-	void run() override {
-		Future<Void> loopFuture = runLoop(this);
-		net2->run();
-	}
+	void run() override { runLoop(this); }
 	ProcessInfo* newProcess(const char* name,
 	                        IPAddress ip,
 	                        uint16_t port,
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 050d91350b..095975d0d1 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2857,7 +2857,7 @@ public:
 		state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(
 		    serverMetadataKeys.begin, IncludeVersion());
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-		state StorageMetadataType data(timer_int());
+		state StorageMetadataType data(StorageMetadataType::currentTime());
 		// printf("------ read metadata %s\n", server->getId().toString().c_str());
 		// read storage metadata
 		loop {
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp
index 115d65be7a..0e08328ff4 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/MoveKeys.actor.cpp
@@ -1151,7 +1151,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 				tr->addReadConflictRange(conflictRange);
 				tr->addWriteConflictRange(conflictRange);
 
-				StorageMetadataType metadata(timer_int());
+				StorageMetadataType metadata(StorageMetadataType::currentTime());
 				metadataMap.set(tr, server.id(), metadata);
 
 				if (SERVER_KNOBS->TSS_HACK_IDENTITY_MAPPING) {
@@ -1521,7 +1521,7 @@ void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<Storag
 	tr.read_conflict_ranges.push_back_deep(arena, allKeys);
 	KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
 	                                                                                     IncludeVersion());
-	StorageMetadataType metadata(timer_int());
+	StorageMetadataType metadata(StorageMetadataType::currentTime());
 
 	for (auto& s : servers) {
 		tr.set(arena, serverTagKeyFor(s.id()), serverTagValue(server_tag[s.id()]));
diff --git a/flow/Arena.cpp b/flow/Arena.cpp
index ae4a0fb2ca..a705f3ecd4 100644
--- a/flow/Arena.cpp
+++ b/flow/Arena.cpp
@@ -352,12 +352,14 @@ ArenaBlock* ArenaBlock::create(int dataSize, Reference<ArenaBlock>& next) {
 			b->bigSize = reqSize;
 			b->bigUsed = sizeof(ArenaBlock);
 
+#if !DEBUG_DETERMINISM
 			if (FLOW_KNOBS && g_allocation_tracing_disabled == 0 &&
 			    nondeterministicRandom()->random01() < (reqSize / FLOW_KNOBS->HUGE_ARENA_LOGGING_BYTES)) {
 				++g_allocation_tracing_disabled;
 				hugeArenaSample(reqSize);
 				--g_allocation_tracing_disabled;
 			}
+#endif
 			g_hugeArenaMemory.fetch_add(reqSize);
 
 			// If the new block has less free space than the old block, make the old block depend on it
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index 0ca5e092af..6aaf295d69 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -527,12 +527,14 @@ void FastAllocator<Size>::getMagazine() {
 	// FIXME: We should be able to allocate larger magazine sizes here if we
 	// detect that the underlying system supports hugepages.  Using hugepages
 	// with smaller-than-2MiB magazine sizes strands memory.  See issue #909.
+#if !DEBUG_DETERMINISM
 	if (FLOW_KNOBS && g_allocation_tracing_disabled == 0 &&
 	    nondeterministicRandom()->random01() < (magazine_size * Size) / FLOW_KNOBS->FAST_ALLOC_LOGGING_BYTES) {
 		++g_allocation_tracing_disabled;
 		TraceEvent("GetMagazineSample").detail("Size", Size).backtrace();
 		--g_allocation_tracing_disabled;
 	}
+#endif
 	block = (void**)::allocate(magazine_size * Size, false);
 #endif
 
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 344fb94c9c..46a99cf439 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -43,7 +43,9 @@ double machineStartTime() {
 
 void systemMonitor() {
 	static StatisticsState statState = StatisticsState();
+#if !DEBUG_DETERMINISM
 	customSystemMonitor("ProcessMetrics", &statState, true);
+#endif
 }
 
 SystemStatistics getSystemStatistics() {
diff --git a/flow/flow.cpp b/flow/flow.cpp
index 42af63210e..d5657c3edb 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -69,7 +69,7 @@ using namespace std::literals;
 
 const std::string_view StackLineage::name = "StackLineage"sv;
 
-#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__AVX__) && !defined(MEMORY_SANITIZER)
+#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__AVX__) && !defined(MEMORY_SANITIZER) && !DEBUG_DETERMINISM
 // For benchmarking; need a version of rte_memcpy that doesn't live in the same compilation unit as the test.
 void* rte_memcpy_noinline(void* __restrict __dest, const void* __restrict __src, size_t __n) {
 	return rte_memcpy(__dest, __src, __n);

From b58739093efd735b71625b9b5db758639499c970 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Fri, 25 Feb 2022 13:18:59 -0800
Subject: [PATCH 121/210] Account for possibility that fopen might fail in mako
 (#6442)

Fix an issue in mako where the stats process would hang if the worker process
was unable to open a file. Previously, the worker process would segfault when
trying to dereference a null FILE* and never increment the stopcount variable
in shared memory. Now we check for a null FILE* and increment stopcount when
the worker process stops regardless of success or failure.

Also don't ignore the return value from mkdir
---
 bindings/c/test/mako/mako.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c
index 7c887dd48a..5ac2340380 100644
--- a/bindings/c/test/mako/mako.c
+++ b/bindings/c/test/mako/mako.c
@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
 #include <math.h>
@@ -7,11 +8,11 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 
 #if defined(__linux__)
 #include <linux/limits.h>
@@ -1350,6 +1351,11 @@ void* worker_thread(void* thread_args) {
 		char str2[1000];
 		sprintf(str2, "%s%d", TEMP_DATA_STORE, *parent_id);
 		rc = mkdir(str2, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+		if (rc < 0) {
+			int ec = errno;
+			fprintf(stderr, "Failed to make directory: %s because %s\n", str2, strerror(ec));
+			goto failExit;
+		}
 		for (op = 0; op < MAX_OP; op++) {
 			if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT || op == OP_TRANSACTION) {
 				FILE* fp;
@@ -1357,6 +1363,11 @@ void* worker_thread(void* thread_args) {
 				strcat(file_name, str2);
 				get_stats_file_name(file_name, worker_id, thread_id, op);
 				fp = fopen(file_name, "w");
+				if (!fp) {
+					int ec = errno;
+					fprintf(stderr, "Failed to open file: %s because %s\n", file_name, strerror(ec));
+					goto failExit;
+				}
 				lat_block_t* temp_block = ((thread_args_t*)thread_args)->block[op];
 				if (is_memory_allocated[op]) {
 					size = stats->latency_samples[op] / LAT_BLOCK_SIZE;
@@ -1376,11 +1387,11 @@ void* worker_thread(void* thread_args) {
 				fclose(fp);
 			}
 		}
-		__sync_fetch_and_add(stopcount, 1);
 	}
 
 	/* fall through */
 failExit:
+	__sync_fetch_and_add(stopcount, 1);
 	for (op = 0; op < MAX_OP; op++) {
 		lat_block_t* curr = ((thread_args_t*)thread_args)->block[op];
 		lat_block_t* prev = NULL;

From 276cf3a402f0268c80c6dad642a41ac05e053361 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Fri, 25 Feb 2022 13:50:41 -0800
Subject: [PATCH 122/210] Fix minor bugs turned up by _GLIBCXX_DEBUG (#6375)

* Fix a benign bug turned up by _GLIBCXX_DEBUG

Just calling std::vector::operator[] with an out of bounds index at all
is technically undefined behavior.

* Fix compilation issue with _GLIBCXX_DEBUG

For some reason std::max with an initializer list isn't constexpr with
_GLIBCXX_DEBUG set
---
 fdbserver/VersionedBTree.actor.cpp |  2 +-
 flow/flat_buffers.h                | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index 13d9c18d15..6666507512 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -8615,7 +8615,7 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/IntIntPair") {
 	DeltaTree<IntIntPair>::Mirror r(tree, &lowerBound, &upperBound);
 
 	DeltaTree2<IntIntPair>* tree2 = (DeltaTree2<IntIntPair>*)new uint8_t[bufferSize];
-	int builtSize2 = tree2->build(bufferSize, &items[0], &items[items.size()], &lowerBound, &upperBound);
+	int builtSize2 = tree2->build(bufferSize, &items[0], &items[0] + items.size(), &lowerBound, &upperBound);
 	ASSERT(builtSize2 <= bufferSize);
 	DeltaTree2<IntIntPair>::DecodeCache cache(lowerBound, upperBound);
 	DeltaTree2<IntIntPair>::Cursor cur2(&cache, tree2);
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index cd44efdd14..cc5c453dc2 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -362,9 +362,18 @@ constexpr size_t AlignToPowerOfTwo(size_t s) {
 	}
 }
 
+template <size_t X, size_t... Xs>
+constexpr size_t max_helper() {
+	if constexpr (sizeof...(Xs) == 0) {
+		return X;
+	} else {
+		return std::max(X, max_helper<Xs...>());
+	}
+}
+
 template <class... Ts>
 constexpr auto align_helper(pack<Ts...>) {
-	return std::max({ size_t{ 1 }, AlignToPowerOfTwo(fb_scalar_size<Ts>)... });
+	return max_helper<1, AlignToPowerOfTwo(fb_scalar_size<Ts>)...>();
 }
 
 template <class... T>

From a61dd41b4552962fa9a09791662587fb819a8a01 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Fri, 25 Feb 2022 15:21:27 +0100
Subject: [PATCH 123/210] Add an empty test to make sure noSim tests succeed

---
 flow/flow.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flow/flow.cpp b/flow/flow.cpp
index d5657c3edb..46e6a2a60b 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -447,3 +447,9 @@ TEST_CASE("/flow/FlatBuffers/Standalone") {
 	}
 	return Void();
 }
+
+// we need to make sure at least one test of each prefix exists, otherwise
+// the noSim test fails if we compile without RocksDB
+TEST_CASE("noSim/noopTest") {
+	return Void();
+}

From 079de5ba57f85e0abb9117e9378bb0b135a3da12 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Fri, 25 Feb 2022 15:25:59 +0100
Subject: [PATCH 124/210] Enable rocksdb unit tests

---
 tests/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 9bf5ac7e34..42e19151b0 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -179,7 +179,9 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/WriteDuringRead.toml)
   add_fdb_test(TEST_FILES fast/WriteDuringReadClean.toml)
   add_fdb_test(TEST_FILES noSim/RandomUnitTests.toml UNIT)
-  add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml IGNORE) # re-enable as needed for RocksDB. Breaks correctness tests if RocksDB is disabled.
+  if (SSD_ROCKSDB_EXPERIMENTAL)
+    add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml IGNORE) # re-enable as needed for RocksDB. Breaks correctness tests if RocksDB is disabled.
+  endif()
   add_fdb_test(TEST_FILES rare/CheckRelocation.toml)
   add_fdb_test(TEST_FILES rare/ClogUnclog.toml)
   add_fdb_test(TEST_FILES rare/CloggedCycleWithKills.toml)

From f7eb66441ded7402d6ca731715fc5486ac38c20b Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Fri, 25 Feb 2022 13:28:29 -0800
Subject: [PATCH 125/210] Try eliminating warnings in macOS and Windows CI
 builds.

MacOS warnings are format warnings, e.g., `format specifies type 'long' but the argument has type 'Version' (aka 'long long')`.
Windows warnings are `ACTOR does not contain a wait() statement`.
---
 bindings/c/test/mako/mako.c                 | 74 ++++++++++-----------
 bindings/c/test/txn_size_test.c             |  8 +--
 documentation/tutorial/tutorial.actor.cpp   |  2 +-
 fdbbackup/backup.actor.cpp                  |  2 +-
 fdbcli/AdvanceVersionCommand.actor.cpp      |  4 +-
 fdbcli/ChangeFeedCommand.actor.cpp          |  4 +-
 fdbcli/StatusCommand.actor.cpp              |  4 +-
 fdbcli/fdbcli.actor.cpp                     |  2 +-
 fdbclient/ConfigKnobs.cpp                   |  2 +-
 fdbclient/FileBackupAgent.actor.cpp         |  4 +-
 fdbclient/ManagementAPI.actor.cpp           |  2 +-
 fdbserver/ConfigDatabaseUnitTests.actor.cpp |  2 +-
 fdbserver/DDTeamCollection.actor.cpp        | 28 ++++----
 fdbserver/KeyValueStoreSQLite.actor.cpp     |  2 +-
 fdbserver/Status.actor.cpp                  |  2 +-
 fdbserver/fdbserver.actor.cpp               |  4 +-
 fdbserver/workloads/Mako.actor.cpp          |  2 +-
 fdbserver/workloads/ReadWrite.actor.cpp     |  2 +-
 flow/Platform.actor.cpp                     |  4 +-
 19 files changed, 75 insertions(+), 79 deletions(-)

diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c
index 5ac2340380..3386622c33 100644
--- a/bindings/c/test/mako/mako.c
+++ b/bindings/c/test/mako/mako.c
@@ -616,7 +616,7 @@ int64_t granule_start_load(const char* filename,
 	// don't seek if offset == 0
 	if (offset && fseek(fp, offset, SEEK_SET)) {
 		// if fseek was non-zero, it failed
-		fprintf(stderr, "ERROR: BG could not seek to %ld in file %s\n", offset, full_fname);
+		fprintf(stderr, "ERROR: BG could not seek to %lld in file %s\n", offset, full_fname);
 		fclose(fp);
 		return -1;
 	}
@@ -626,7 +626,7 @@ int64_t granule_start_load(const char* filename,
 	fclose(fp);
 
 	if (readSize != length) {
-		fprintf(stderr, "ERROR: BG could not read %ld bytes from file: %s\n", length, full_fname);
+		fprintf(stderr, "ERROR: BG could not read %lld bytes from file: %s\n", length, full_fname);
 		return -1;
 	}
 
@@ -637,7 +637,7 @@ int64_t granule_start_load(const char* filename,
 uint8_t* granule_get_load(int64_t loadId, void* userContext) {
 	BGLocalFileContext* context = (BGLocalFileContext*)userContext;
 	if (context->data_by_id[loadId] == 0) {
-		fprintf(stderr, "ERROR: BG loadId invalid for get_load: %ld\n", loadId);
+		fprintf(stderr, "ERROR: BG loadId invalid for get_load: %lld\n", loadId);
 		return 0;
 	}
 	return context->data_by_id[loadId];
@@ -646,7 +646,7 @@ uint8_t* granule_get_load(int64_t loadId, void* userContext) {
 void granule_free_load(int64_t loadId, void* userContext) {
 	BGLocalFileContext* context = (BGLocalFileContext*)userContext;
 	if (context->data_by_id[loadId] == 0) {
-		fprintf(stderr, "ERROR: BG loadId invalid for free_load: %ld\n", loadId);
+		fprintf(stderr, "ERROR: BG loadId invalid for free_load: %lld\n", loadId);
 	}
 	free(context->data_by_id[loadId]);
 	context->data_by_id[loadId] = 0;
@@ -1120,7 +1120,7 @@ int run_workload(FDBTransaction* transaction,
 					if (tracetimer == dotrace) {
 						fdb_error_t err;
 						tracetimer = 0;
-						snprintf(traceid, 32, "makotrace%019ld", total_xacts);
+						snprintf(traceid, 32, "makotrace%019lld", total_xacts);
 						fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
 						err = fdb_transaction_set_option(transaction,
 						                                 FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
@@ -1284,7 +1284,7 @@ void* worker_thread(void* thread_args) {
 	}
 
 	fprintf(debugme,
-	        "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%lu (tid:%lu)\n",
+	        "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%lu (tid:%llu)\n",
 	        worker_id,
 	        args->num_processes,
 	        thread_id,
@@ -2251,9 +2251,9 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0) {
 			uint64_t ops_total_diff = ops_total[op] - ops_total_prev[op];
-			printf("%" STR(STATS_FIELD_WIDTH) "lu ", ops_total_diff);
+			printf("%" STR(STATS_FIELD_WIDTH) "llu ", ops_total_diff);
 			if (fp) {
-				fprintf(fp, "\"%s\": %lu,", get_ops_name(op), ops_total_diff);
+				fprintf(fp, "\"%s\": %llu,", get_ops_name(op), ops_total_diff);
 			}
 			errors_diff[op] = errors_total[op] - errors_total_prev[op];
 			print_err = (errors_diff[op] > 0);
@@ -2281,7 +2281,7 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 		printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors");
 		for (op = 0; op < MAX_OP; op++) {
 			if (args->txnspec.ops[op][OP_COUNT] > 0) {
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", errors_diff[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", errors_diff[op]);
 				if (fp) {
 					fprintf(fp, ",\"errors\": %.2f", conflicts_diff);
 				}
@@ -2430,10 +2430,10 @@ void print_report(mako_args_t* args,
 			break;
 		}
 	}
-	printf("Total Xacts:      %8lu\n", totalxacts);
-	printf("Total Conflicts:  %8lu\n", conflicts);
-	printf("Total Errors:     %8lu\n", totalerrors);
-	printf("Overall TPS:      %8lu\n\n", totalxacts * 1000000000 / duration_nsec);
+	printf("Total Xacts:      %8llu\n", totalxacts);
+	printf("Total Conflicts:  %8llu\n", conflicts);
+	printf("Total Errors:     %8llu\n", totalerrors);
+	printf("Overall TPS:      %8llu\n\n", totalxacts * 1000000000 / duration_nsec);
 
 	if (fp) {
 		fprintf(fp, "\"results\": {");
@@ -2441,10 +2441,10 @@ void print_report(mako_args_t* args,
 		fprintf(fp, "\"totalProcesses\": %d,", args->num_processes);
 		fprintf(fp, "\"totalThreads\": %d,", args->num_threads);
 		fprintf(fp, "\"targetTPS\": %d,", args->tpsmax);
-		fprintf(fp, "\"totalXacts\": %lu,", totalxacts);
-		fprintf(fp, "\"totalConflicts\": %lu,", conflicts);
-		fprintf(fp, "\"totalErrors\": %lu,", totalerrors);
-		fprintf(fp, "\"overallTPS\": %lu,", totalxacts * 1000000000 / duration_nsec);
+		fprintf(fp, "\"totalXacts\": %llu,", totalxacts);
+		fprintf(fp, "\"totalConflicts\": %llu,", conflicts);
+		fprintf(fp, "\"totalErrors\": %llu,", totalerrors);
+		fprintf(fp, "\"overallTPS\": %llu,", totalxacts * 1000000000 / duration_nsec);
 	}
 
 	/* per-op stats */
@@ -2457,14 +2457,14 @@ void print_report(mako_args_t* args,
 	}
 	for (op = 0; op < MAX_OP; op++) {
 		if ((args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
-			printf("%" STR(STATS_FIELD_WIDTH) "lu ", ops_total[op]);
+			printf("%" STR(STATS_FIELD_WIDTH) "llu ", ops_total[op]);
 			if (fp) {
 				if (first_op) {
 					first_op = 0;
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %lu", get_ops_name(op), ops_total[op]);
+				fprintf(fp, "\"%s\": %llu", get_ops_name(op), ops_total[op]);
 			}
 		}
 	}
@@ -2486,14 +2486,14 @@ void print_report(mako_args_t* args,
 	first_op = 1;
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) {
-			printf("%" STR(STATS_FIELD_WIDTH) "lu ", errors_total[op]);
+			printf("%" STR(STATS_FIELD_WIDTH) "llu ", errors_total[op]);
 			if (fp) {
 				if (first_op) {
 					first_op = 0;
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %lu", get_ops_name(op), errors_total[op]);
+				fprintf(fp, "\"%s\": %llu", get_ops_name(op), errors_total[op]);
 			}
 		}
 	}
@@ -2511,7 +2511,7 @@ void print_report(mako_args_t* args,
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
 			if (lat_total[op]) {
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_samples[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_samples[op]);
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2521,7 +2521,7 @@ void print_report(mako_args_t* args,
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %lu", get_ops_name(op), lat_samples[op]);
+				fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_samples[op]);
 			}
 		}
 	}
@@ -2538,14 +2538,14 @@ void print_report(mako_args_t* args,
 			if (lat_min[op] == -1) {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_min[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_min[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), lat_min[op]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_min[op]);
 				}
 			}
 		}
@@ -2561,14 +2561,14 @@ void print_report(mako_args_t* args,
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
 			if (lat_total[op]) {
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_total[op] / lat_samples[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_total[op] / lat_samples[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), lat_total[op] / lat_samples[op]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_total[op] / lat_samples[op]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2588,14 +2588,14 @@ void print_report(mako_args_t* args,
 			if (lat_max[op] == 0) {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_max[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_max[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), lat_max[op]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_max[op]);
 				}
 			}
 		}
@@ -2646,14 +2646,14 @@ void print_report(mako_args_t* args,
 				} else {
 					median = (dataPoints[op][num_points[op] / 2] + dataPoints[op][num_points[op] / 2 - 1]) >> 1;
 				}
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", median);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", median);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), median);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), median);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2676,14 +2676,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_95pct = ((float)(num_points[op]) * 0.95) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_95pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_95pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), dataPoints[op][point_95pct]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_95pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2706,14 +2706,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_99pct = ((float)(num_points[op]) * 0.99) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_99pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_99pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), dataPoints[op][point_99pct]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_99pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2736,14 +2736,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_99_9pct = ((float)(num_points[op]) * 0.999) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_99_9pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_99_9pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %lu", get_ops_name(op), dataPoints[op][point_99_9pct]);
+					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_99_9pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c
index b8be90ceb1..9205364b15 100644
--- a/bindings/c/test/txn_size_test.c
+++ b/bindings/c/test/txn_size_test.c
@@ -67,25 +67,25 @@ void runTests(struct ResultSet* rs) {
 	fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %ld\n", i, sizes[i]);
+	printf("size %d: %lld\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %ld\n", i, sizes[i]);
+	printf("size %d: %lld\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_clear(tr, keys[i], KEY_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %ld\n", i, sizes[i]);
+	printf("size %d: %lld\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %ld\n", i, sizes[i]);
+	printf("size %d: %lld\n", i, sizes[i]);
 	i++;
 
 	for (j = 0; j + 1 < i; j++) {
diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index 326f0246f2..a4c7289048 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -413,7 +413,7 @@ ACTOR Future<Void> logThroughput(int64_t* v, Key* next) {
 	loop {
 		state int64_t last = *v;
 		wait(delay(1));
-		printf("throughput: %ld bytes/s, next: %s\n", *v - last, printable(*next).c_str());
+		printf("throughput: %lld bytes/s, next: %s\n", *v - last, printable(*next).c_str());
 	}
 }
 
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index f2a1356c35..4590b0f548 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2754,7 +2754,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			reportBackupQueryError(operationId,
 			                       result,
 			                       errorMessage =
-			                           format("the specified restorable version %ld is not valid", restoreVersion));
+			                           format("the specified restorable version %lld is not valid", restoreVersion));
 			return Void();
 		}
 		Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter));
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp
index 0a5902c6e6..7a86e9072a 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@@ -40,7 +40,7 @@ ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vect
 	} else {
 		state Version v;
 		int n = 0;
-		if (sscanf(tokens[1].toString().c_str(), "%ld%n", &v, &n) != 1 || n != tokens[1].size()) {
+		if (sscanf(tokens[1].toString().c_str(), "%lld%n", &v, &n) != 1 || n != tokens[1].size()) {
 			printUsage(tokens[0]);
 			return false;
 		} else {
@@ -53,7 +53,7 @@ ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vect
 						tr->set(advanceVersionSpecialKey, boost::lexical_cast<std::string>(v));
 						wait(safeThreadFutureToFuture(tr->commit()));
 					} else {
-						printf("Current read version is %ld\n", rv);
+						printf("Current read version is %lld\n", rv);
 						return true;
 					}
 				} catch (Error& e) {
diff --git a/fdbcli/ChangeFeedCommand.actor.cpp b/fdbcli/ChangeFeedCommand.actor.cpp
index e08eadb37e..ba696186c0 100644
--- a/fdbcli/ChangeFeedCommand.actor.cpp
+++ b/fdbcli/ChangeFeedCommand.actor.cpp
@@ -115,7 +115,7 @@ ACTOR Future<bool> changeFeedCommandActor(Database localDb, std::vector<StringRe
 		Version end = std::numeric_limits<Version>::max();
 		if (tokens.size() > 3) {
 			int n = 0;
-			if (sscanf(tokens[3].toString().c_str(), "%ld%n", &begin, &n) != 1 || n != tokens[3].size()) {
+			if (sscanf(tokens[3].toString().c_str(), "%lld%n", &begin, &n) != 1 || n != tokens[3].size()) {
 				printUsage(tokens[0]);
 				return false;
 			}
@@ -168,7 +168,7 @@ ACTOR Future<bool> changeFeedCommandActor(Database localDb, std::vector<StringRe
 		}
 		Version v;
 		int n = 0;
-		if (sscanf(tokens[3].toString().c_str(), "%ld%n", &v, &n) != 1 || n != tokens[3].size()) {
+		if (sscanf(tokens[3].toString().c_str(), "%lld%n", &v, &n) != 1 || n != tokens[3].size()) {
 			printUsage(tokens[0]);
 			return false;
 		} else {
diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp
index be30c1d47e..ea6bcb5293 100644
--- a/fdbcli/StatusCommand.actor.cpp
+++ b/fdbcli/StatusCommand.actor.cpp
@@ -705,12 +705,12 @@ void printStatus(StatusObjectReader statusObj,
 										}
 									}
 									outputString += format(
-									    "  %s log epoch: %ld begin: %ld end: %s, missing "
+									    "  %s log epoch: %lld begin: %lld end: %s, missing "
 									    "log interfaces(id,address): %s\n",
 									    current ? "Current" : "Old",
 									    epoch,
 									    beginVersion,
-									    endVersion == invalidVersion ? "(unknown)" : format("%ld", endVersion).c_str(),
+									    endVersion == invalidVersion ? "(unknown)" : format("%lld", endVersion).c_str(),
 									    missing_log_interfaces.c_str());
 								}
 							}
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index fe93f236c5..cae38f0bb7 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1615,7 +1615,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					} else {
 						Version v = wait(makeInterruptable(
 						    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
-						printf("%ld\n", v);
+						printf("%lld\n", v);
 					}
 					continue;
 				}
diff --git a/fdbclient/ConfigKnobs.cpp b/fdbclient/ConfigKnobs.cpp
index f2a1338bf8..9318c949a3 100644
--- a/fdbclient/ConfigKnobs.cpp
+++ b/fdbclient/ConfigKnobs.cpp
@@ -96,7 +96,7 @@ public:
 
 struct ToStringFunc {
 	std::string operator()(int v) const { return format("int:%d", v); }
-	std::string operator()(int64_t v) const { return format("int64_t:%ld", v); }
+	std::string operator()(int64_t v) const { return format("int64_t:%lld", v); }
 	std::string operator()(bool v) const { return format("bool:%d", v); }
 	std::string operator()(ValueRef v) const { return "string:" + v.toString(); }
 	std::string operator()(double v) const { return format("double:%lf", v); }
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index d431c763c5..251e0ab9a4 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -87,7 +87,7 @@ std::string secondsToTimeFormat(int64_t seconds) {
 	else if (seconds >= 60)
 		return format("%.2f minute(s)", seconds / 60.0);
 	else
-		return format("%ld second(s)", seconds);
+		return format("%lld second(s)", seconds);
 }
 
 const Key FileBackupAgent::keyLastRestorable = LiteralStringRef("last_restorable");
@@ -5183,7 +5183,7 @@ public:
 						else
 							statusText += "The initial snapshot is still running.\n";
 
-						statusText += format("\nDetails:\n LogBytes written - %ld\n RangeBytes written - %ld\n "
+						statusText += format("\nDetails:\n LogBytes written - %lld\n RangeBytes written - %lld\n "
 						                     "Last complete log version and timestamp        - %s, %s\n "
 						                     "Last complete snapshot version and timestamp   - %s, %s\n "
 						                     "Current Snapshot start version and timestamp   - %s, %s\n "
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 99ebea516d..78a2443849 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2201,7 +2201,7 @@ ACTOR Future<Void> advanceVersion(Database cx, Version v) {
 				tr.set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned()));
 				wait(tr.commit());
 			} else {
-				printf("Current read version is %ld\n", rv);
+				printf("Current read version is %lld\n", rv);
 				return Void();
 			}
 		} catch (Error& e) {
diff --git a/fdbserver/ConfigDatabaseUnitTests.actor.cpp b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
index e5104d51bd..26c1a3e925 100644
--- a/fdbserver/ConfigDatabaseUnitTests.actor.cpp
+++ b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
@@ -61,7 +61,7 @@ class WriteToTransactionEnvironment {
 	Version lastWrittenVersion{ 0 };
 
 	static Value longToValue(int64_t v) {
-		auto s = format("%ld", v);
+		auto s = format("%lld", v);
 		return StringRef(reinterpret_cast<uint8_t const*>(s.c_str()), s.size());
 	}
 
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 095975d0d1..68656c8604 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -5209,13 +5209,13 @@ public:
 		return Void();
 	}
 
-	ACTOR static Future<Void> AddAllTeams_isExhaustive() {
+	static void AddAllTeams_isExhaustive() {
 		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
 		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-		state int processSize = 10;
-		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
+		int processSize = 10;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
 
 		int result = collection->addTeamsBestOf(200, desiredTeams, maxTeams);
 
@@ -5223,24 +5223,20 @@ public:
 		// The maximum number of available server teams with machine locality constraint is 120 - 40, because
 		// the 40 (5*4*2) server teams whose servers come from the same machine are invalid.
 		ASSERT(result == 80);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> AddAllTeams_withLimit() {
+	static void AddAllTeams_withLimit() {
 		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
 		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-		state int processSize = 10;
-		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		int processSize = 10;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
 
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(3, policy, processSize);
 
 		int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
 
 		ASSERT(result >= 10);
-
-		return Void();
 	}
 
 	ACTOR static Future<Void> AddTeamsBestOf_SkippingBusyServers() {
@@ -5665,12 +5661,12 @@ TEST_CASE("DataDistribution/AddTeamsBestOf/NotUseMachineID") {
 }
 
 TEST_CASE("DataDistribution/AddAllTeams/isExhaustive") {
-	wait(DDTeamCollectionUnitTest::AddAllTeams_isExhaustive());
+	DDTeamCollectionUnitTest::AddAllTeams_isExhaustive();
 	return Void();
 }
 
 TEST_CASE("/DataDistribution/AddAllTeams/withLimit") {
-	wait(DDTeamCollectionUnitTest::AddAllTeams_withLimit());
+	DDTeamCollectionUnitTest::AddAllTeams_withLimit();
 	return Void();
 }
 
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 571c7d49c9..554f10c501 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -2359,7 +2359,7 @@ ACTOR Future<Void> KVFileDump(std::string filename) {
 		k = keyAfter(kv[kv.size() - 1].key);
 	}
 	fflush(stdout);
-	fprintf(stderr, "Counted: %ld\n", count);
+	fprintf(stderr, "Counted: %lld\n", count);
 
 	if (store->getError().isError())
 		wait(store->getError());
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 1c4ab387d8..c34830311d 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -1038,7 +1038,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 			if (ssLag[address] >= 60) {
 				messages.push_back(JsonString::makeMessage(
 				    "storage_server_lagging",
-				    format("Storage server lagging by %ld seconds.", (int64_t)ssLag[address]).c_str()));
+				    format("Storage server lagging by %lld seconds.", (int64_t)ssLag[address]).c_str()));
 			}
 
 			// Store the message array into the status object that represents the worker process
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 1a2a91f77b..0d28953d04 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -1432,8 +1432,8 @@ private:
 				// parameter
 				knobs.emplace_back(
 				    "page_cache_4k",
-				    format("%ld", ti.get() / 4096 * 4096)); // The cache holds 4K pages, so we can truncate this to the
-				                                            // next smaller multiple of 4K.
+				    format("%lld", ti.get() / 4096 * 4096)); // The cache holds 4K pages, so we can truncate this to the
+				                                             // next smaller multiple of 4K.
 				break;
 			case OPT_BUGGIFY:
 				if (!strcmp(args.OptionArg(), "on"))
diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp
index 6864ffe5e2..0db3bc62ec 100644
--- a/fdbserver/workloads/Mako.actor.cpp
+++ b/fdbserver/workloads/Mako.actor.cpp
@@ -183,7 +183,7 @@ struct MakoWorkload : TestWorkload {
 			auto ratesItr = ratesAtKeyCounts.begin();
 			for (; ratesItr != ratesAtKeyCounts.end(); ratesItr++) {
 				m.emplace_back(
-				    format("%ld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
+				    format("%lld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
 			}
 		}
 		// benchmark
diff --git a/fdbserver/workloads/ReadWrite.actor.cpp b/fdbserver/workloads/ReadWrite.actor.cpp
index 4eae887c77..669d0d4265 100644
--- a/fdbserver/workloads/ReadWrite.actor.cpp
+++ b/fdbserver/workloads/ReadWrite.actor.cpp
@@ -317,7 +317,7 @@ struct ReadWriteWorkload : KVWorkload {
 
 		std::vector<std::pair<uint64_t, double>>::iterator ratesItr = ratesAtKeyCounts.begin();
 		for (; ratesItr != ratesAtKeyCounts.end(); ratesItr++)
-			m.emplace_back(format("%ld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
+			m.emplace_back(format("%lld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
 	}
 
 	Value randomValue() {
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index d42c8e721c..9c61bfe7ff 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -3826,7 +3826,7 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	ASSERT(request[LiteralStringRef("SwapTotal:")] == 25165820);
 	ASSERT(request[LiteralStringRef("SwapFree:")] == 23680228);
 	for (auto& item : request) {
-		printf("%s:%ld\n", item.first.toString().c_str(), item.second);
+		printf("%s:%lld\n", item.first.toString().c_str(), item.second);
 	}
 
 	printf("UnitTest flow/Platform/getMemoryInfo 2\n");
@@ -3877,7 +3877,7 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	ASSERT(request[LiteralStringRef("SwapTotal:")] == 0);
 	ASSERT(request[LiteralStringRef("SwapFree:")] == 0);
 	for (auto& item : request) {
-		printf("%s:%ld\n", item.first.toString().c_str(), item.second);
+		printf("%s:%lld\n", item.first.toString().c_str(), item.second);
 	}
 
 	return Void();

From 233c918ffbd8706df137a79f94928c12ee5eb030 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Fri, 25 Feb 2022 15:49:23 -0800
Subject: [PATCH 126/210] Replace printf() and fprintf() with fmt::print().

---
 documentation/tutorial/tutorial.actor.cpp | 3 ++-
 fdbbackup/backup.actor.cpp                | 2 +-
 fdbcli/AdvanceVersionCommand.actor.cpp    | 4 ++--
 fdbcli/fdbcli.actor.cpp                   | 2 +-
 fdbclient/ManagementAPI.actor.cpp         | 2 +-
 fdbserver/KeyValueStoreSQLite.actor.cpp   | 3 ++-
 flow/Platform.actor.cpp                   | 5 +++--
 7 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index a4c7289048..8fdb32b0c2 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -19,6 +19,7 @@
 * limitations under the License.
 */
 
+#include "contrib/fmt-8.0.1/include/fmt/format.h"
 #include "flow/flow.h"
 #include "flow/Platform.h"
 #include "flow/DeterministicRandom.h"
@@ -413,7 +414,7 @@ ACTOR Future<Void> logThroughput(int64_t* v, Key* next) {
 	loop {
 		state int64_t last = *v;
 		wait(delay(1));
-		printf("throughput: %lld bytes/s, next: %s\n", *v - last, printable(*next).c_str());
+		fmt::print("throughput: {} bytes/s, next: {}\n", *v - last, printable(*next).c_str());
 	}
 }
 
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 4590b0f548..e3cdc01ce7 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -3081,7 +3081,7 @@ static void addKeyRange(std::string optionValue, Standalone<VectorRef<KeyRangeRe
 
 			// Too many keys
 		default:
-			fprintf(stderr, "ERROR: Invalid key range identified with %ld keys", tokens.size());
+			fmt::print(stderr, "ERROR: Invalid key range identified with {} keys", tokens.size());
 			throw invalid_option_value();
 			break;
 		}
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp
index 7a86e9072a..9a916d946f 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include "boost/lexical_cast.hpp"
-
+#include "contrib/fmt-8.0.1/include/fmt/format.h"
 #include "fdbcli/fdbcli.actor.h"
 
 #include "fdbclient/IClientApi.h"
@@ -53,7 +53,7 @@ ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vect
 						tr->set(advanceVersionSpecialKey, boost::lexical_cast<std::string>(v));
 						wait(safeThreadFutureToFuture(tr->commit()));
 					} else {
-						printf("Current read version is %lld\n", rv);
+						fmt::print("Current read version is {}\n", rv);
 						return true;
 					}
 				} catch (Error& e) {
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index cae38f0bb7..f7720f9efb 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1615,7 +1615,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					} else {
 						Version v = wait(makeInterruptable(
 						    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
-						printf("%lld\n", v);
+						fmt::print("{}\n", v);
 					}
 					continue;
 				}
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 78a2443849..bd016538a7 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2201,7 +2201,7 @@ ACTOR Future<Void> advanceVersion(Database cx, Version v) {
 				tr.set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned()));
 				wait(tr.commit());
 			} else {
-				printf("Current read version is %lld\n", rv);
+				fmt::print("Current read version is {}\n", rv);
 				return Void();
 			}
 		} catch (Error& e) {
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 554f10c501..4ab37463c5 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #define SQLITE_THREADSAFE 0 // also in sqlite3.amalgamation.c!
+#include "contrib/fmt-8.0.1/include/fmt/format.h"
 #include "flow/crc32c.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/CoroFlow.h"
@@ -2359,7 +2360,7 @@ ACTOR Future<Void> KVFileDump(std::string filename) {
 		k = keyAfter(kv[kv.size() - 1].key);
 	}
 	fflush(stdout);
-	fprintf(stderr, "Counted: %lld\n", count);
+	fmt::print(stderr, "Counted: {}\n", count);
 
 	if (store->getError().isError())
 		wait(store->getError());
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 9c61bfe7ff..3a7da8da52 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -26,6 +26,7 @@
 #endif
 
 #include <errno.h>
+#include "contrib/fmt-8.0.1/include/fmt/format.h"
 #include "flow/Platform.h"
 #include "flow/Platform.actor.h"
 #include "flow/Arena.h"
@@ -3826,7 +3827,7 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	ASSERT(request[LiteralStringRef("SwapTotal:")] == 25165820);
 	ASSERT(request[LiteralStringRef("SwapFree:")] == 23680228);
 	for (auto& item : request) {
-		printf("%s:%lld\n", item.first.toString().c_str(), item.second);
+		fmt::print("{}:{}\n", item.first.toString().c_str(), item.second);
 	}
 
 	printf("UnitTest flow/Platform/getMemoryInfo 2\n");
@@ -3877,7 +3878,7 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	ASSERT(request[LiteralStringRef("SwapTotal:")] == 0);
 	ASSERT(request[LiteralStringRef("SwapFree:")] == 0);
 	for (auto& item : request) {
-		printf("%s:%lld\n", item.first.toString().c_str(), item.second);
+		fmt::print("{}:{}\n", item.first.toString().c_str(), item.second);
 	}
 
 	return Void();

From 90927cffe989477898a3768a7f76f4e059e86097 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Sat, 26 Feb 2022 15:04:54 -0800
Subject: [PATCH 127/210] Set disableHostname=true in CycleTestRestart
 downgrade test

---
 tests/restarting/to_7.0.0/CycleTestRestart-1.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/restarting/to_7.0.0/CycleTestRestart-1.txt b/tests/restarting/to_7.0.0/CycleTestRestart-1.txt
index bcd80fa1e2..78c3c64e04 100644
--- a/tests/restarting/to_7.0.0/CycleTestRestart-1.txt
+++ b/tests/restarting/to_7.0.0/CycleTestRestart-1.txt
@@ -1,6 +1,8 @@
 storageEngineExcludeTypes=-1,-2
 maxTLogVersion=6
 disableTss=true
+disableHostname=true
+
 testTitle=Clogged
     clearAfterTest=false
     testName=Cycle
@@ -30,4 +32,4 @@ testTitle=Clogged
 
     testName=SaveAndKill
     restartInfoLocation=simfdb/restartInfo.ini
-    testDuration=10.0
\ No newline at end of file
+    testDuration=10.0

From d559caefeaa4e83bc681a720e15e52a99884ee9e Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Tue, 22 Feb 2022 11:29:38 -0800
Subject: [PATCH 128/210] Use centos:7 for pkg_tester

yum update with just `centos` seems to have problems
---
 contrib/pkg_tester/test_fdb_pkgs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/pkg_tester/test_fdb_pkgs.py b/contrib/pkg_tester/test_fdb_pkgs.py
index 3b6c4855e1..28f0841838 100644
--- a/contrib/pkg_tester/test_fdb_pkgs.py
+++ b/contrib/pkg_tester/test_fdb_pkgs.py
@@ -147,7 +147,7 @@ def centos_image_with_fdb_helper(versioned: bool) -> Iterator[Optional[Image]]:
     container = None
     image = None
     try:
-        container = Container("centos", initd=True)
+        container = Container("centos:7", initd=True)
         for rpm in rpms:
             container.copy_to(rpm, "/opt")
         container.run(["bash", "-c", "yum update -y"])

From 24f37aceb7a9c473a6ca711b6bd3724c26f9c08f Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Tue, 22 Feb 2022 13:31:34 -0800
Subject: [PATCH 129/210] Avoid conflict between versioned client/server rpms

Building locally this seems to cause a conflict between the versioned
client and server rpms. Not sure why this doesn't always show up.
---
 cmake/InstallLayout.cmake | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake
index 61af9df658..8da0fce34f 100644
--- a/cmake/InstallLayout.cmake
+++ b/cmake/InstallLayout.cmake
@@ -303,7 +303,9 @@ set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
   "/etc/rc.d/init.d"
   "/usr/lib/pkgconfig"
   "/usr/lib/foundationdb"
-  "/usr/lib/cmake")
+  "/usr/lib/cmake"
+  "/usr/lib/foundationdb-${FDB_VERSION}/etc/foundationdb"
+  )
 set(CPACK_RPM_DEBUGINFO_PACKAGE ${GENERATE_DEBUG_PACKAGES})
 #set(CPACK_RPM_BUILD_SOURCE_FDB_INSTALL_DIRS_PREFIX /usr/src)
 set(CPACK_RPM_COMPONENT_INSTALL ON)

From 2e8bce04c6f52e331d9b52c78baf06b9ebf332b3 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Tue, 22 Feb 2022 13:32:51 -0800
Subject: [PATCH 130/210] Remove brittle fdbcli help text snapshot test

This seems to be more trouble than value so far.
---
 contrib/pkg_tester/test_fdb_pkgs.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/contrib/pkg_tester/test_fdb_pkgs.py b/contrib/pkg_tester/test_fdb_pkgs.py
index 28f0841838..db8a1ae27b 100644
--- a/contrib/pkg_tester/test_fdb_pkgs.py
+++ b/contrib/pkg_tester/test_fdb_pkgs.py
@@ -237,10 +237,6 @@ def test_write(linux_container: Container, snapshot):
     assert snapshot == linux_container.run(["fdbcli", "--exec", "get x"])
 
 
-def test_fdbcli_help_text(linux_container: Container, snapshot):
-    assert snapshot == linux_container.run(["fdbcli", "--help"])
-
-
 def test_execstack_permissions_libfdb_c(linux_container: Container, snapshot):
     linux_container.run(["ldconfig"])
     assert snapshot == linux_container.run(

From 847cff72b748ef51df9eab47c5d6e17b31862f4a Mon Sep 17 00:00:00 2001
From: zhenfeng yang <92761100+sfc-gh-zyang@users.noreply.github.com>
Date: Mon, 28 Feb 2022 15:00:30 -0800
Subject: [PATCH 131/210] Add a target to generate profile (#6396)

add a target to generate profile
---
 cmake/ConfigureCompiler.cmake |  2 +-
 contrib/generate_profile.sh   | 37 +++++++++++++++++++++++++++++++++++
 fdbmonitor/CMakeLists.txt     | 10 ++++++++++
 3 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100755 contrib/generate_profile.sh

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 83ed75e873..f05ca279be 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -309,7 +309,7 @@ else()
       if (PROFILE_INSTR_GENERATE)
           message(FATAL_ERROR "Can't set both PROFILE_INSTR_GENERATE and PROFILE_INSTR_USE")
       endif()
-      add_compile_options(-Wno-error=profile-instr-out-of-date)
+      add_compile_options(-Wno-error=profile-instr-out-of-date -Wno-error=profile-instr-unprofiled)
       add_compile_options(-fprofile-instr-use=${PROFILE_INSTR_USE})
       add_link_options(-fprofile-instr-use=${PROFILE_INSTR_USE})
     endif()
diff --git a/contrib/generate_profile.sh b/contrib/generate_profile.sh
new file mode 100755
index 0000000000..9b6a446bd2
--- /dev/null
+++ b/contrib/generate_profile.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+if [ $# -eq 0 ] || [ $# -gt 2 ]
+  then
+    echo "Usage: generate_profile.sh Path_Of_Foundation_Build_Directory Storage_Engine"
+    exit 1
+fi
+fdbdir=$1
+storage_engine='ssd'
+if [ $# -eq 2 ]
+  then
+    storage_engine=$2
+fi
+
+export LD_LIBRARY_PATH=$fdbdir/lib:$LD_LIBRARY_PATH
+export FDB_CLUSTER_FILE=$fdbdir/fdb.cluster
+export LLVM_PROFILE_FILE=$fdbdir/sandbox/fdb-%p.profraw
+$fdbdir/bin/fdbmonitor --conffile $fdbdir/sandbox/foundationdb.conf --lockfile $fdbdir/sandbox/fdbmonitor.pid &
+# This profile will be ignored
+export LLVM_PROFILE_FILE=$fdbdir/sandbox/cli-%m.profraw
+$fdbdir/bin/fdbcli -C $fdbdir/fdb.cluster --exec "configure new $storage_engine single"
+export LLVM_PROFILE_FILE=$fdbdir/sandbox/mako-build-%m.profraw
+$fdbdir/bin/mako -p 64 -t 1 --keylen 32 --vallen 16 --mode build --rows 10000  --trace  --trace_format json
+export LLVM_PROFILE_FILE=$fdbdir/sandbox/mako-run-%m.profraw
+$fdbdir/bin/mako -p 1 -t 2 --keylen 32 --vallen 16 --mode run --rows 10000 --transaction grvg7i2gr1:48cr1:48 --seconds 60 --trace $fdbdir/sandbox/logs --trace_format json
+
+# Shutdown fdbserver to trigger profile dumping
+fdbmonitor_pid=$(cat $fdbdir/sandbox/fdbmonitor.pid)
+fdbserver_pid=$(cat /proc/$fdbmonitor_pid/task/$fdbmonitor_pid/children)
+gdb --batch --eval-command 'call (void)exit(0)' --pid $fdbserver_pid
+
+# Clean up
+kill -9 $fdbmonitor_pid
+
+# Profile for server
+llvm-profdata merge -output=$fdbdir/fdb.profdata $fdbdir/sandbox/fdb-*.profraw
+# Profile for client
+llvm-profdata merge -output=$fdbdir/mako.profdata $fdbdir/sandbox/mako-*.profraw
diff --git a/fdbmonitor/CMakeLists.txt b/fdbmonitor/CMakeLists.txt
index 70f57cb00b..5d624e8191 100644
--- a/fdbmonitor/CMakeLists.txt
+++ b/fdbmonitor/CMakeLists.txt
@@ -35,3 +35,13 @@ add_custom_target(start_sandbox
                                              --lockfile ${CMAKE_BINARY_DIR}/sandbox/fdbmonitor.lock)
 
 add_dependencies(start_sandbox fdbmonitor fdbserver)
+
+if(NOT EXISTS ${CMAKE_BINARY_DIR}/contrib/generate_profile.sh)
+  configure_file(${CMAKE_SOURCE_DIR}/contrib/generate_profile.sh
+    ${CMAKE_BINARY_DIR}/contrib/generate_profile.sh)
+endif()
+
+add_custom_target(generate_profile
+  COMMAND  ${CMAKE_BINARY_DIR}/contrib/generate_profile.sh ${CMAKE_BINARY_DIR})
+
+add_dependencies(generate_profile fdbmonitor fdbserver mako fdbcli)

From aadd706a0a4b5c50e5779056d7a0c99ae95ce8b9 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 28 Feb 2022 14:54:37 -0800
Subject: [PATCH 132/210] Port back addressed comments for fixing get key
 location issue

---
 fdbclient/NativeAPI.actor.cpp | 77 ++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 42 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index d4abc5cf8a..ea88b59747 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1600,7 +1600,8 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
 
 void DatabaseContext::setFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
 	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
-		failedEndpointsOnHealthyServersInfo[endpoint] = { /*startTime=*/now(), /*lastRefreshTime=*/now() };
+		failedEndpointsOnHealthyServersInfo[endpoint] =
+		EndpointFailureInfo{ startTime : now(), lastRefreshTime : now() };
 	}
 }
 
@@ -1609,7 +1610,6 @@ void DatabaseContext::updateFailedEndpointRefreshTime(const Endpoint& endpoint)
 		// The endpoint is not failed. Nothing to update.
 		return;
 	}
-
 	failedEndpointsOnHealthyServersInfo[endpoint].lastRefreshTime = now();
 }
 
@@ -2477,6 +2477,35 @@ ACTOR Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation_intern
 	}
 }
 
+// Checks if `endpoint` is failed on a healthy server or not. Returns true if we need to refresh the location cache for
+// the endpoint.
+bool checkOnlyEndpointFailed(const Database& cx, const Endpoint& endpoint) {
+	if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
+		// This endpoint is failed, but the server is still healthy. There are two cases this can happen:
+		//    - There is a recent bounce in the cluster where the endpoints in SSes get updated.
+		//    - The SS is failed and terminated on a server, but the server is kept running.
+		// To account for the first case, we invalidate the cache and issue GetKeyLocation requests to the proxy to
+		// update the cache with the new SS points. However, if the failure is caused by the second case, the
+		// requested key location will continue to be the failed endpoint until the data movement is finished. But
+		// every read will generate a GetKeyLocation request to the proxies (and still getting the failed endpoint
+		// back), which may overload the proxy and affect data movement speed. Therefore, we only refresh the
+		// location cache for short period of time, and after the initial grace period that we keep retrying
+		// resolving key location, we will slow it down to resolve it only once every
+		// `LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL`.
+		cx->setFailedEndpointOnHealthyServer(endpoint);
+		const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
+		ASSERT(failureInfo.present());
+		if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
+		    now() - failureInfo.get().lastRefreshTime > CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
+			cx->updateFailedEndpointRefreshTime(endpoint);
+			return true;
+		}
+	} else {
+		cx->clearFailedEndpointOnHealthyServer(endpoint);
+	}
+	return false;
+}
+
 template <class F>
 Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database const& cx,
                                                                     Key const& key,
@@ -2493,36 +2522,13 @@ Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database con
 
 	bool onlyEndpointFailedAndNeedRefresh = false;
 	for (int i = 0; i < ssi.second->size(); i++) {
-		const Endpoint& endpoint = ssi.second->get(i, member).getEndpoint();
-		if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
-			// This endpoint is failed, but the server is still healthy. There are two cases this can happen:
-			//    - There is a recent bounce in the cluster where the endpoints in SSes get updated.
-			//    - The SS is failed and terminated on a server, but the server is kept running.
-			// To account for the first case, we invalidate the cache and issue GetKeyLocation requests to the proxy to
-			// update the cache with the new SS points. However, if the failure is caused by the second case, the
-			// requested key location will continue to be the failed endpoint until the data movement is finished. But
-			// every read will generate a GetKeyLocation request to the proxies (and still getting the failed endpoint
-			// back), which may overload the proxy and affect data movement speed. Therefore, we only refresh the
-			// location cache for short period of time, and after the initial grace period that we keep retrying
-			// resolving key location, we will slow it down to resolve it only once every
-			// `LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL`.
-			cx->setFailedEndpointOnHealthyServer(endpoint);
-			const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
-			ASSERT(failureInfo.present());
-			if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
-			    now() - failureInfo.get().lastRefreshTime >
-			        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
-				onlyEndpointFailedAndNeedRefresh = true;
-				cx->updateFailedEndpointRefreshTime(ssi.second->get(i, member).getEndpoint());
-			}
-		} else {
-			cx->clearFailedEndpointOnHealthyServer(endpoint);
+		if (checkOnlyEndpointFailed(cx, ssi.second->get(i, member).getEndpoint())) {
+			onlyEndpointFailedAndNeedRefresh = true;
 		}
 	}
 
 	if (onlyEndpointFailedAndNeedRefresh) {
 		cx->invalidateCache(key);
-
 		// Refresh the cache with a new getKeyLocations made to proxies.
 		return getKeyLocation_internal(cx, key, spanID, debugID, useProvisionalProxies, isBackward);
 	}
@@ -2611,21 +2617,8 @@ Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLoc
 	for (const auto& [range, locInfo] : locations) {
 		bool onlyEndpointFailedAndNeedRefresh = false;
 		for (int i = 0; i < locInfo->size(); i++) {
-			const Endpoint& endpoint = locInfo->get(i, member).getEndpoint();
-			if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
-				// Please refer to `getKeyLocation` about why we keep track of endpoint failure starting and refreshing
-				// time.
-				cx->setFailedEndpointOnHealthyServer(endpoint);
-				const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
-				ASSERT(failureInfo.present());
-				if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
-				    now() - failureInfo.get().lastRefreshTime >
-				        CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
-					onlyEndpointFailedAndNeedRefresh = true;
-					cx->updateFailedEndpointRefreshTime(locInfo->get(i, member).getEndpoint());
-				}
-			} else {
-				cx->clearFailedEndpointOnHealthyServer(endpoint);
+			if (checkOnlyEndpointFailed(cx, locInfo->get(i, member).getEndpoint())) {
+				onlyEndpointFailedAndNeedRefresh = true;
 			}
 		}
 

From 02e396274f850329a02168ef809b5b8b749c3aee Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Mon, 28 Feb 2022 16:34:59 -0800
Subject: [PATCH 133/210] Fix the compile error on NativeAPI.actor.cpp

---
 fdbclient/NativeAPI.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index ea88b59747..e9489c8bf7 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1601,7 +1601,7 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
 void DatabaseContext::setFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
 	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
 		failedEndpointsOnHealthyServersInfo[endpoint] =
-		EndpointFailureInfo{ startTime : now(), lastRefreshTime : now() };
+		EndpointFailureInfo{ .startTime = now(), .lastRefreshTime = now() };
 	}
 }
 

From 5e39da77418e24131480f0fb98d95ff4f81b4718 Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Mon, 28 Feb 2022 16:44:10 -0800
Subject: [PATCH 134/210] fixup! Reformat the source

---
 fdbclient/NativeAPI.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index e9489c8bf7..90d9e85cfd 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1601,7 +1601,7 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
 void DatabaseContext::setFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
 	if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
 		failedEndpointsOnHealthyServersInfo[endpoint] =
-		EndpointFailureInfo{ .startTime = now(), .lastRefreshTime = now() };
+		    EndpointFailureInfo{ .startTime = now(), .lastRefreshTime = now() };
 	}
 }
 

From fa2eee827965dcbc4fd8f07e79ebba05427d3856 Mon Sep 17 00:00:00 2001
From: Renxuan Wang <renxuan@apple.com>
Date: Mon, 28 Feb 2022 18:19:52 -0800
Subject: [PATCH 135/210] Use PRId64 and PRIu64 to replace lld and llu.

This fixes https://github.com/apple/foundationdb/pull/6148#issuecomment-1054575608.
---
 bindings/c/test/mako/mako.c            | 76 +++++++++++++-------------
 bindings/c/test/txn_size_test.c        |  8 +--
 fdbcli/AdvanceVersionCommand.actor.cpp |  2 +-
 fdbcli/ChangeFeedCommand.actor.cpp     |  4 +-
 4 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c
index 3386622c33..b08a3b2366 100644
--- a/bindings/c/test/mako/mako.c
+++ b/bindings/c/test/mako/mako.c
@@ -2,8 +2,8 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
+#include <inttypes.h>
 #include <math.h>
-#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -616,7 +616,7 @@ int64_t granule_start_load(const char* filename,
 	// don't seek if offset == 0
 	if (offset && fseek(fp, offset, SEEK_SET)) {
 		// if fseek was non-zero, it failed
-		fprintf(stderr, "ERROR: BG could not seek to %lld in file %s\n", offset, full_fname);
+		fprintf(stderr, "ERROR: BG could not seek to %" PRId64 " in file %s\n", offset, full_fname);
 		fclose(fp);
 		return -1;
 	}
@@ -626,7 +626,7 @@ int64_t granule_start_load(const char* filename,
 	fclose(fp);
 
 	if (readSize != length) {
-		fprintf(stderr, "ERROR: BG could not read %lld bytes from file: %s\n", length, full_fname);
+		fprintf(stderr, "ERROR: BG could not read %" PRId64 " bytes from file: %s\n", length, full_fname);
 		return -1;
 	}
 
@@ -637,7 +637,7 @@ int64_t granule_start_load(const char* filename,
 uint8_t* granule_get_load(int64_t loadId, void* userContext) {
 	BGLocalFileContext* context = (BGLocalFileContext*)userContext;
 	if (context->data_by_id[loadId] == 0) {
-		fprintf(stderr, "ERROR: BG loadId invalid for get_load: %lld\n", loadId);
+		fprintf(stderr, "ERROR: BG loadId invalid for get_load: %" PRId64 "\n", loadId);
 		return 0;
 	}
 	return context->data_by_id[loadId];
@@ -646,7 +646,7 @@ uint8_t* granule_get_load(int64_t loadId, void* userContext) {
 void granule_free_load(int64_t loadId, void* userContext) {
 	BGLocalFileContext* context = (BGLocalFileContext*)userContext;
 	if (context->data_by_id[loadId] == 0) {
-		fprintf(stderr, "ERROR: BG loadId invalid for free_load: %lld\n", loadId);
+		fprintf(stderr, "ERROR: BG loadId invalid for free_load: %" PRId64 "\n", loadId);
 	}
 	free(context->data_by_id[loadId]);
 	context->data_by_id[loadId] = 0;
@@ -1120,7 +1120,7 @@ int run_workload(FDBTransaction* transaction,
 					if (tracetimer == dotrace) {
 						fdb_error_t err;
 						tracetimer = 0;
-						snprintf(traceid, 32, "makotrace%019lld", total_xacts);
+						snprintf(traceid, 32, "makotrace%019" PRId64, total_xacts);
 						fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
 						err = fdb_transaction_set_option(transaction,
 						                                 FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
@@ -1284,7 +1284,7 @@ void* worker_thread(void* thread_args) {
 	}
 
 	fprintf(debugme,
-	        "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%lu (tid:%llu)\n",
+	        "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%lu (tid:%" PRIu64 ")\n",
 	        worker_id,
 	        args->num_processes,
 	        thread_id,
@@ -2251,9 +2251,9 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0) {
 			uint64_t ops_total_diff = ops_total[op] - ops_total_prev[op];
-			printf("%" STR(STATS_FIELD_WIDTH) "llu ", ops_total_diff);
+			printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", ops_total_diff);
 			if (fp) {
-				fprintf(fp, "\"%s\": %llu,", get_ops_name(op), ops_total_diff);
+				fprintf(fp, "\"%s\": %" PRIu64 ",", get_ops_name(op), ops_total_diff);
 			}
 			errors_diff[op] = errors_total[op] - errors_total_prev[op];
 			print_err = (errors_diff[op] > 0);
@@ -2281,7 +2281,7 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 		printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors");
 		for (op = 0; op < MAX_OP; op++) {
 			if (args->txnspec.ops[op][OP_COUNT] > 0) {
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", errors_diff[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", errors_diff[op]);
 				if (fp) {
 					fprintf(fp, ",\"errors\": %.2f", conflicts_diff);
 				}
@@ -2430,10 +2430,10 @@ void print_report(mako_args_t* args,
 			break;
 		}
 	}
-	printf("Total Xacts:      %8llu\n", totalxacts);
-	printf("Total Conflicts:  %8llu\n", conflicts);
-	printf("Total Errors:     %8llu\n", totalerrors);
-	printf("Overall TPS:      %8llu\n\n", totalxacts * 1000000000 / duration_nsec);
+	printf("Total Xacts:      %8" PRIu64 "\n", totalxacts);
+	printf("Total Conflicts:  %8" PRIu64 "\n", conflicts);
+	printf("Total Errors:     %8" PRIu64 "\n", totalerrors);
+	printf("Overall TPS:      %8" PRIu64 "\n\n", totalxacts * 1000000000 / duration_nsec);
 
 	if (fp) {
 		fprintf(fp, "\"results\": {");
@@ -2441,10 +2441,10 @@ void print_report(mako_args_t* args,
 		fprintf(fp, "\"totalProcesses\": %d,", args->num_processes);
 		fprintf(fp, "\"totalThreads\": %d,", args->num_threads);
 		fprintf(fp, "\"targetTPS\": %d,", args->tpsmax);
-		fprintf(fp, "\"totalXacts\": %llu,", totalxacts);
-		fprintf(fp, "\"totalConflicts\": %llu,", conflicts);
-		fprintf(fp, "\"totalErrors\": %llu,", totalerrors);
-		fprintf(fp, "\"overallTPS\": %llu,", totalxacts * 1000000000 / duration_nsec);
+		fprintf(fp, "\"totalXacts\": %" PRIu64 ",", totalxacts);
+		fprintf(fp, "\"totalConflicts\": %" PRIu64 ",", conflicts);
+		fprintf(fp, "\"totalErrors\": %" PRIu64 ",", totalerrors);
+		fprintf(fp, "\"overallTPS\": %" PRIu64 ",", totalxacts * 1000000000 / duration_nsec);
 	}
 
 	/* per-op stats */
@@ -2457,14 +2457,14 @@ void print_report(mako_args_t* args,
 	}
 	for (op = 0; op < MAX_OP; op++) {
 		if ((args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
-			printf("%" STR(STATS_FIELD_WIDTH) "llu ", ops_total[op]);
+			printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", ops_total[op]);
 			if (fp) {
 				if (first_op) {
 					first_op = 0;
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %llu", get_ops_name(op), ops_total[op]);
+				fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), ops_total[op]);
 			}
 		}
 	}
@@ -2486,14 +2486,14 @@ void print_report(mako_args_t* args,
 	first_op = 1;
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) {
-			printf("%" STR(STATS_FIELD_WIDTH) "llu ", errors_total[op]);
+			printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", errors_total[op]);
 			if (fp) {
 				if (first_op) {
 					first_op = 0;
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %llu", get_ops_name(op), errors_total[op]);
+				fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), errors_total[op]);
 			}
 		}
 	}
@@ -2511,7 +2511,7 @@ void print_report(mako_args_t* args,
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
 			if (lat_total[op]) {
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_samples[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", lat_samples[op]);
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2521,7 +2521,7 @@ void print_report(mako_args_t* args,
 				} else {
 					fprintf(fp, ",");
 				}
-				fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_samples[op]);
+				fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), lat_samples[op]);
 			}
 		}
 	}
@@ -2538,14 +2538,14 @@ void print_report(mako_args_t* args,
 			if (lat_min[op] == -1) {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_min[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", lat_min[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_min[op]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), lat_min[op]);
 				}
 			}
 		}
@@ -2561,14 +2561,14 @@ void print_report(mako_args_t* args,
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
 			if (lat_total[op]) {
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_total[op] / lat_samples[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", lat_total[op] / lat_samples[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_total[op] / lat_samples[op]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), lat_total[op] / lat_samples[op]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2588,14 +2588,14 @@ void print_report(mako_args_t* args,
 			if (lat_max[op] == 0) {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", lat_max[op]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", lat_max[op]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), lat_max[op]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), lat_max[op]);
 				}
 			}
 		}
@@ -2646,14 +2646,14 @@ void print_report(mako_args_t* args,
 				} else {
 					median = (dataPoints[op][num_points[op] / 2] + dataPoints[op][num_points[op] / 2 - 1]) >> 1;
 				}
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", median);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", median);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), median);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), median);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2676,14 +2676,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_95pct = ((float)(num_points[op]) * 0.95) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_95pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", dataPoints[op][point_95pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_95pct]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), dataPoints[op][point_95pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2706,14 +2706,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_99pct = ((float)(num_points[op]) * 0.99) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_99pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", dataPoints[op][point_99pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_99pct]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), dataPoints[op][point_99pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
@@ -2736,14 +2736,14 @@ void print_report(mako_args_t* args,
 			}
 			if (lat_total[op]) {
 				point_99_9pct = ((float)(num_points[op]) * 0.999) - 1;
-				printf("%" STR(STATS_FIELD_WIDTH) "llu ", dataPoints[op][point_99_9pct]);
+				printf("%" STR(STATS_FIELD_WIDTH) PRIu64 " ", dataPoints[op][point_99_9pct]);
 				if (fp) {
 					if (first_op) {
 						first_op = 0;
 					} else {
 						fprintf(fp, ",");
 					}
-					fprintf(fp, "\"%s\": %llu", get_ops_name(op), dataPoints[op][point_99_9pct]);
+					fprintf(fp, "\"%s\": %" PRIu64, get_ops_name(op), dataPoints[op][point_99_9pct]);
 				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c
index 9205364b15..1fb08eb3f5 100644
--- a/bindings/c/test/txn_size_test.c
+++ b/bindings/c/test/txn_size_test.c
@@ -67,25 +67,25 @@ void runTests(struct ResultSet* rs) {
 	fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %lld\n", i, sizes[i]);
+	printf("size %d: %" PRId64 "\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %lld\n", i, sizes[i]);
+	printf("size %d: %" PRId64 "\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_clear(tr, keys[i], KEY_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %lld\n", i, sizes[i]);
+	printf("size %d: %" PRId64 "\n", i, sizes[i]);
 	i++;
 
 	fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
 	e = getSize(rs, tr, sizes + i);
 	checkError(e, "transaction get size", rs);
-	printf("size %d: %lld\n", i, sizes[i]);
+	printf("size %d: %" PRId64 "\n", i, sizes[i]);
 	i++;
 
 	for (j = 0; j + 1 < i; j++) {
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp
index 9a916d946f..3e6e4de22d 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@@ -40,7 +40,7 @@ ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vect
 	} else {
 		state Version v;
 		int n = 0;
-		if (sscanf(tokens[1].toString().c_str(), "%lld%n", &v, &n) != 1 || n != tokens[1].size()) {
+		if (sscanf(tokens[1].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[1].size()) {
 			printUsage(tokens[0]);
 			return false;
 		} else {
diff --git a/fdbcli/ChangeFeedCommand.actor.cpp b/fdbcli/ChangeFeedCommand.actor.cpp
index ba696186c0..94415be921 100644
--- a/fdbcli/ChangeFeedCommand.actor.cpp
+++ b/fdbcli/ChangeFeedCommand.actor.cpp
@@ -115,7 +115,7 @@ ACTOR Future<bool> changeFeedCommandActor(Database localDb, std::vector<StringRe
 		Version end = std::numeric_limits<Version>::max();
 		if (tokens.size() > 3) {
 			int n = 0;
-			if (sscanf(tokens[3].toString().c_str(), "%lld%n", &begin, &n) != 1 || n != tokens[3].size()) {
+			if (sscanf(tokens[3].toString().c_str(), "%" PRId64 "%n", &begin, &n) != 1 || n != tokens[3].size()) {
 				printUsage(tokens[0]);
 				return false;
 			}
@@ -168,7 +168,7 @@ ACTOR Future<bool> changeFeedCommandActor(Database localDb, std::vector<StringRe
 		}
 		Version v;
 		int n = 0;
-		if (sscanf(tokens[3].toString().c_str(), "%lld%n", &v, &n) != 1 || n != tokens[3].size()) {
+		if (sscanf(tokens[3].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[3].size()) {
 			printUsage(tokens[0]);
 			return false;
 		} else {

From 97aec45ce8144b7096774ddaf5c531c1ee9dc7d1 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 1 Mar 2022 08:51:50 -0800
Subject: [PATCH 136/210] Fix formatting issues caused by clang-format

---
 fdbserver/workloads/WriteDuringRead.actor.cpp | 1232 +++++++++--------
 1 file changed, 620 insertions(+), 612 deletions(-)

diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index d18b6d0fd9..c41bf68f4a 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -444,645 +444,653 @@ struct WriteDuringReadWorkload : TestWorkload {
 			state int changeNum = self->changeCount[key];
 			state Optional<Value> memRes = self->memoryGet(&self->memoryDatabase, key);
 			*memLimit -= memRes.expectedSize();
-			choose{ when(wait(tr->watch(key))){
-				if (changeNum == self->changeCount[key]){ TraceEvent(SevError, "WDRWatchWrongResult", randomID)
-				                                              .detail("Reason", "Triggered without changing")
-				                                              .detail("Key", printable(key))
-				                                              .detail("Value", changeNum)
-				                                              .detail("DuringCommit", *doingCommit);
-		}
-	}
-	when(wait(self->finished.onTrigger())) {
-		Optional<Value> memRes2 = self->memoryGet(&self->memoryDatabase, key);
-		if (memRes != memRes2) {
-			TraceEvent(SevError, "WDRWatchWrongResult", randomID)
-			    .detail("Reason", "Changed without triggering")
-			    .detail("Key", printable(key))
-			    .detail("Value1", printable(memRes))
-			    .detail("Value2", printable(memRes2));
-		}
-	}
-}* memLimit += memRes.expectedSize();
 
-return Void();
-}
-catch (Error& e) {
-	// check for transaction cancelled if the watch was not committed
-	//TraceEvent("WDRWatchError", randomID).error(e,true);
-	if (e.code() == error_code_used_during_commit) {
-		ASSERT(*doingCommit);
-		return Void();
-	} else if (e.code() == error_code_transaction_cancelled)
-		return Void();
-	throw;
-}
-}
-
-ACTOR Future<Void> commitAndUpdateMemory(ReadYourWritesTransaction* tr,
-                                         WriteDuringReadWorkload* self,
-                                         bool* cancelled,
-                                         bool readYourWritesDisabled,
-                                         bool snapshotRYWDisabled,
-                                         bool readAheadDisabled,
-                                         bool useBatchPriority,
-                                         bool* doingCommit,
-                                         double* startTime,
-                                         Key timebombStr) {
-	// state UID randomID = nondeterministicRandom()->randomUniqueID();
-	//TraceEvent("WDRCommit", randomID);
-	try {
-		if (!readYourWritesDisabled && !*cancelled) {
-			KeyRangeMap<bool> transactionConflicts;
-			tr->getWriteConflicts(&transactionConflicts);
-
-			auto transactionRanges = transactionConflicts.ranges();
-			auto addedRanges = self->addedConflicts.ranges();
-			auto transactionIter = transactionRanges.begin();
-			auto addedIter = addedRanges.begin();
-
-			bool failed = false;
-			while (transactionIter != transactionRanges.end() && addedIter != addedRanges.end()) {
-				if (transactionIter->begin() != addedIter->begin() || transactionIter->value() != addedIter->value()) {
-					TraceEvent(SevError, "WriteConflictError")
-					    .detail("TransactionKey", printable(transactionIter->begin()))
-					    .detail("AddedKey", printable(addedIter->begin()))
-					    .detail("TransactionVal", transactionIter->value())
-					    .detail("AddedVal", addedIter->value());
-					failed = true;
-				}
-				++transactionIter;
-				++addedIter;
-			}
-
-			if (transactionIter != transactionRanges.end() || addedIter != addedRanges.end()) {
-				failed = true;
-			}
-
-			if (failed) {
-				TraceEvent(SevError, "WriteConflictRangeError").log();
-				for (transactionIter = transactionRanges.begin(); transactionIter != transactionRanges.end();
-				     ++transactionIter) {
-					TraceEvent("WCRTransaction")
-					    .detail("Range", printable(transactionIter.range()))
-					    .detail("Value", transactionIter.value());
-				}
-				for (addedIter = addedRanges.begin(); addedIter != addedRanges.end(); ++addedIter) {
-					TraceEvent("WCRAdded")
-					    .detail("Range", printable(addedIter.range()))
-					    .detail("Value", addedIter.value());
-				}
-			}
-		}
-
-		state std::map<Key, Value> committedDB = self->memoryDatabase;
-		*doingCommit = true;
-		wait(tr->commit());
-		*doingCommit = false;
-		self->finished.trigger();
-
-		if (readYourWritesDisabled)
-			tr->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
-		if (snapshotRYWDisabled)
-			tr->setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
-		if (readAheadDisabled)
-			tr->setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
-		if (useBatchPriority)
-			tr->setOption(FDBTransactionOptions::PRIORITY_BATCH);
-		if (self->useSystemKeys)
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr->addWriteConflictRange(self->conflictRange);
-		self->addedConflicts.insert(allKeys, false);
-		self->addedConflicts.insert(self->conflictRange, true);
-		*startTime = now();
-		tr->setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
-
-		//TraceEvent("WDRCommitSuccess", randomID).detail("CommittedVersion", tr->getCommittedVersion());
-		self->lastCommittedDatabase = committedDB;
-
-		return Void();
-	} catch (Error& e) {
-		//TraceEvent("WDRCommitCancelled", randomID).error(e,true);
-		if (e.code() == error_code_actor_cancelled || e.code() == error_code_transaction_cancelled ||
-		    e.code() == error_code_used_during_commit)
-			*cancelled = true;
-		if (e.code() == error_code_actor_cancelled || e.code() == error_code_transaction_cancelled)
-			throw commit_unknown_result();
-		if (e.code() == error_code_transaction_too_old)
-			throw not_committed();
-		throw;
-	}
-}
-
-Value getRandomValue() {
-	return Value(std::string(deterministicRandom()->randomInt(valueSizeRange.first, valueSizeRange.second + 1), 'x'));
-}
-
-ACTOR Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
-	state double startTime = now();
-	loop {
-		state int i = 0;
-		state int keysPerBatch =
-		    std::min<int64_t>(1000,
-		                      1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 6 /
-		                              (self->getKeyForIndex(self->nodes).size() + self->valueSizeRange.second));
-		self->memoryDatabase = std::map<Key, Value>();
-		for (; i < self->nodes; i += keysPerBatch) {
-			state Transaction tr(cx);
-			loop {
-				if (now() - startTime > self->testDuration)
-					return Void();
-				try {
-					if (i == 0) {
-						tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-						tr.addWriteConflictRange(
-						    allKeys); // To prevent a write only transaction whose commit was previously cancelled from
-						              // being reordered after this transaction
-						tr.clear(normalKeys);
-					}
-					if (self->useSystemKeys)
-						tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-
-					int end = std::min(self->nodes, i + keysPerBatch);
-					tr.clear(KeyRangeRef(self->getKeyForIndex(i), self->getKeyForIndex(end)));
-					self->memoryDatabase.erase(self->memoryDatabase.lower_bound(self->getKeyForIndex(i)),
-					                           self->memoryDatabase.lower_bound(self->getKeyForIndex(end)));
-
-					for (int j = i; j < end; j++) {
-						if (deterministicRandom()->random01() < self->initialKeyDensity) {
-							Key key = self->getKeyForIndex(j);
-							if (key.size() <= (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-							                                                    : CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
-								Value value = self->getRandomValue();
-								value = value.substr(0, std::min<int>(value.size(), CLIENT_KNOBS->VALUE_SIZE_LIMIT));
-								self->memoryDatabase[key] = value;
-								tr.set(key, value);
-							}
+			// This if block prevents formatting issues with clang-format
+			if (1) {
+				choose {
+					when(wait(tr->watch(key))) {
+						if (changeNum == self->changeCount[key]) {
+							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
+							    .detail("Reason", "Triggered without changing")
+							    .detail("Key", printable(key))
+							    .detail("Value", changeNum)
+							    .detail("DuringCommit", *doingCommit);
+						}
+					}
+					when(wait(self->finished.onTrigger())) {
+						Optional<Value> memRes2 = self->memoryGet(&self->memoryDatabase, key);
+						if (memRes != memRes2) {
+							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
+							    .detail("Reason", "Changed without triggering")
+							    .detail("Key", printable(key))
+							    .detail("Value1", printable(memRes))
+							    .detail("Value2", printable(memRes2));
 						}
 					}
-					wait(tr.commit());
-					//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr.getCommittedVersion());
-					break;
-				} catch (Error& e) {
-					wait(tr.onError(e));
 				}
 			}
+			*memLimit += memRes.expectedSize();
+
+			return Void();
+		} catch (Error& e) {
+			// check for transaction cancelled if the watch was not committed
+			//TraceEvent("WDRWatchError", randomID).error(e,true);
+			if (e.code() == error_code_used_during_commit) {
+				ASSERT(*doingCommit);
+				return Void();
+			} else if (e.code() == error_code_transaction_cancelled)
+				return Void();
+			throw;
 		}
-		self->lastCommittedDatabase = self->memoryDatabase;
-		self->addedConflicts.insert(allKeys, false);
-		//TraceEvent("WDRInit");
+	}
+
+	ACTOR Future<Void> commitAndUpdateMemory(ReadYourWritesTransaction* tr,
+	                                         WriteDuringReadWorkload* self,
+	                                         bool* cancelled,
+	                                         bool readYourWritesDisabled,
+	                                         bool snapshotRYWDisabled,
+	                                         bool readAheadDisabled,
+	                                         bool useBatchPriority,
+	                                         bool* doingCommit,
+	                                         double* startTime,
+	                                         Key timebombStr) {
+		// state UID randomID = nondeterministicRandom()->randomUniqueID();
+		//TraceEvent("WDRCommit", randomID);
+		try {
+			if (!readYourWritesDisabled && !*cancelled) {
+				KeyRangeMap<bool> transactionConflicts;
+				tr->getWriteConflicts(&transactionConflicts);
+
+				auto transactionRanges = transactionConflicts.ranges();
+				auto addedRanges = self->addedConflicts.ranges();
+				auto transactionIter = transactionRanges.begin();
+				auto addedIter = addedRanges.begin();
+
+				bool failed = false;
+				while (transactionIter != transactionRanges.end() && addedIter != addedRanges.end()) {
+					if (transactionIter->begin() != addedIter->begin() ||
+					    transactionIter->value() != addedIter->value()) {
+						TraceEvent(SevError, "WriteConflictError")
+						    .detail("TransactionKey", printable(transactionIter->begin()))
+						    .detail("AddedKey", printable(addedIter->begin()))
+						    .detail("TransactionVal", transactionIter->value())
+						    .detail("AddedVal", addedIter->value());
+						failed = true;
+					}
+					++transactionIter;
+					++addedIter;
+				}
+
+				if (transactionIter != transactionRanges.end() || addedIter != addedRanges.end()) {
+					failed = true;
+				}
+
+				if (failed) {
+					TraceEvent(SevError, "WriteConflictRangeError").log();
+					for (transactionIter = transactionRanges.begin(); transactionIter != transactionRanges.end();
+					     ++transactionIter) {
+						TraceEvent("WCRTransaction")
+						    .detail("Range", printable(transactionIter.range()))
+						    .detail("Value", transactionIter.value());
+					}
+					for (addedIter = addedRanges.begin(); addedIter != addedRanges.end(); ++addedIter) {
+						TraceEvent("WCRAdded")
+						    .detail("Range", printable(addedIter.range()))
+						    .detail("Value", addedIter.value());
+					}
+				}
+			}
+
+			state std::map<Key, Value> committedDB = self->memoryDatabase;
+			*doingCommit = true;
+			wait(tr->commit());
+			*doingCommit = false;
+			self->finished.trigger();
+
+			if (readYourWritesDisabled)
+				tr->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+			if (snapshotRYWDisabled)
+				tr->setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
+			if (readAheadDisabled)
+				tr->setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
+			if (useBatchPriority)
+				tr->setOption(FDBTransactionOptions::PRIORITY_BATCH);
+			if (self->useSystemKeys)
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->addWriteConflictRange(self->conflictRange);
+			self->addedConflicts.insert(allKeys, false);
+			self->addedConflicts.insert(self->conflictRange, true);
+			*startTime = now();
+			tr->setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
+
+			//TraceEvent("WDRCommitSuccess", randomID).detail("CommittedVersion", tr->getCommittedVersion());
+			self->lastCommittedDatabase = committedDB;
+
+			return Void();
+		} catch (Error& e) {
+			//TraceEvent("WDRCommitCancelled", randomID).error(e,true);
+			if (e.code() == error_code_actor_cancelled || e.code() == error_code_transaction_cancelled ||
+			    e.code() == error_code_used_during_commit)
+				*cancelled = true;
+			if (e.code() == error_code_actor_cancelled || e.code() == error_code_transaction_cancelled)
+				throw commit_unknown_result();
+			if (e.code() == error_code_transaction_too_old)
+				throw not_committed();
+			throw;
+		}
+	}
+
+	Value getRandomValue() {
+		return Value(
+		    std::string(deterministicRandom()->randomInt(valueSizeRange.first, valueSizeRange.second + 1), 'x'));
+	}
+
+	ACTOR Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
+		state double startTime = now();
+		loop {
+			state int i = 0;
+			state int keysPerBatch =
+			    std::min<int64_t>(1000,
+			                      1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 6 /
+			                              (self->getKeyForIndex(self->nodes).size() + self->valueSizeRange.second));
+			self->memoryDatabase = std::map<Key, Value>();
+			for (; i < self->nodes; i += keysPerBatch) {
+				state Transaction tr(cx);
+				loop {
+					if (now() - startTime > self->testDuration)
+						return Void();
+					try {
+						if (i == 0) {
+							tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+							tr.addWriteConflictRange(
+							    allKeys); // To prevent a write only transaction whose commit was previously cancelled
+							              // from being reordered after this transaction
+							tr.clear(normalKeys);
+						}
+						if (self->useSystemKeys)
+							tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+						int end = std::min(self->nodes, i + keysPerBatch);
+						tr.clear(KeyRangeRef(self->getKeyForIndex(i), self->getKeyForIndex(end)));
+						self->memoryDatabase.erase(self->memoryDatabase.lower_bound(self->getKeyForIndex(i)),
+						                           self->memoryDatabase.lower_bound(self->getKeyForIndex(end)));
+
+						for (int j = i; j < end; j++) {
+							if (deterministicRandom()->random01() < self->initialKeyDensity) {
+								Key key = self->getKeyForIndex(j);
+								if (key.size() <= (key.startsWith(systemKeys.begin)
+								                       ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+								                       : CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
+									Value value = self->getRandomValue();
+									value =
+									    value.substr(0, std::min<int>(value.size(), CLIENT_KNOBS->VALUE_SIZE_LIMIT));
+									self->memoryDatabase[key] = value;
+									tr.set(key, value);
+								}
+							}
+						}
+						wait(tr.commit());
+						//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr.getCommittedVersion());
+						break;
+					} catch (Error& e) {
+						wait(tr.onError(e));
+					}
+				}
+			}
+			self->lastCommittedDatabase = self->memoryDatabase;
+			self->addedConflicts.insert(allKeys, false);
+			//TraceEvent("WDRInit");
+
+			loop {
+				wait(delay(now() - startTime > self->slowModeStart ||
+				                   (g_network->isSimulated() && g_simulator.speedUpSimulation)
+				               ? 1.0
+				               : 0.1));
+				try {
+					wait(self->randomTransaction(
+					    (self->useExtraDB && deterministicRandom()->random01() < 0.5) ? self->extraDB : cx,
+					    self,
+					    startTime));
+				} catch (Error& e) {
+					if (e.code() != error_code_not_committed)
+						throw;
+					break;
+				}
+				if (now() - startTime > self->testDuration)
+					return Void();
+			}
+		}
+	}
+
+	Key getRandomKey() { return getKeyForIndex(deterministicRandom()->randomInt(0, nodes)); }
+
+	Key getKeyForIndex(int idx) {
+		idx += minNode;
+		if (adjacentKeys) {
+			return Key(idx ? keyPrefix + std::string(idx, '\x00') : "");
+		} else {
+			return Key(keyPrefix + format("%010d", idx));
+		}
+	}
+
+	Key versionStampKeyForIndex(int idx) {
+		Key result = KeyRef(getKeyForIndex(idx).toString() + std::string(14, '\x00'));
+		int32_t pos = deterministicRandom()->randomInt(0, result.size() - 13);
+		pos = littleEndian32(pos);
+		uint8_t* data = mutateString(result);
+		memcpy(data + result.size() - sizeof(int32_t), &pos, sizeof(int32_t));
+		return result;
+	}
+
+	Key getRandomVersionStampKey() { return versionStampKeyForIndex(deterministicRandom()->randomInt(0, nodes)); }
+
+	KeySelector getRandomKeySelector() {
+		int scale = 1 << deterministicRandom()->randomInt(0, 14);
+		return KeySelectorRef(
+		    getRandomKey(), deterministicRandom()->random01() < 0.5, deterministicRandom()->randomInt(-scale, scale));
+	}
+
+	GetRangeLimits getRandomLimits() {
+		int kind = deterministicRandom()->randomInt(0, 3);
+		return GetRangeLimits(
+		    (kind & 1) ? GetRangeLimits::ROW_LIMIT_UNLIMITED
+		               : deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(1, 10)),
+		    (kind & 2) ? GetRangeLimits::BYTE_LIMIT_UNLIMITED
+		               : deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(1, 15)));
+	}
+
+	KeyRange getRandomRange(int sizeLimit) {
+		int startLocation = deterministicRandom()->randomInt(0, nodes);
+		int scale = deterministicRandom()->randomInt(
+		    0, deterministicRandom()->randomInt(2, 5) * deterministicRandom()->randomInt(2, 5));
+		int endLocation = startLocation + deterministicRandom()->randomInt(
+		                                      0, 1 + std::min(sizeLimit, std::min(nodes - startLocation, 1 << scale)));
+
+		return KeyRangeRef(getKeyForIndex(startLocation), getKeyForIndex(endLocation));
+	}
+
+	Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type) {
+		Arena arena;
+		if (type == MutationRef::SetValue)
+			return value;
+		else if (type == MutationRef::AddValue)
+			return doLittleEndianAdd(existingValue, value, arena);
+		else if (type == MutationRef::AppendIfFits)
+			return doAppendIfFits(existingValue, value, arena);
+		else if (type == MutationRef::And)
+			return doAndV2(existingValue, value, arena);
+		else if (type == MutationRef::Or)
+			return doOr(existingValue, value, arena);
+		else if (type == MutationRef::Xor)
+			return doXor(existingValue, value, arena);
+		else if (type == MutationRef::Max)
+			return doMax(existingValue, value, arena);
+		else if (type == MutationRef::Min)
+			return doMinV2(existingValue, value, arena);
+		else if (type == MutationRef::ByteMin)
+			return doByteMin(existingValue, value, arena);
+		else if (type == MutationRef::ByteMax)
+			return doByteMax(existingValue, value, arena);
+		ASSERT(false);
+		return Value();
+	}
+
+	ACTOR Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self, double testStartTime) {
+		state ReadYourWritesTransaction tr(cx);
+		state bool readYourWritesDisabled = deterministicRandom()->random01() < 0.5;
+		state bool readAheadDisabled = deterministicRandom()->random01() < 0.5;
+		state bool snapshotRYWDisabled = deterministicRandom()->random01() < 0.5;
+		state bool useBatchPriority = deterministicRandom()->random01() < 0.5;
+		state int64_t timebomb =
+		    (FLOW_KNOBS->MAX_BUGGIFIED_DELAY == 0.0 && deterministicRandom()->random01() < 0.01)
+		        ? deterministicRandom()->randomInt64(1, 6000)
+		        : 0; // timebomb check can fail incorrectly if simulation injects delay longer than the timebomb
+		state std::vector<Future<Void>> operations;
+		state ActorCollection commits(false);
+		state std::vector<Future<Void>> watches;
+		state int changeNum = 1;
+		state bool doingCommit = false;
+		state int waitLocation = 0;
+		state double startTime = now();
+
+		state bool disableGetKey = BUGGIFY;
+		state bool disableGetRange = BUGGIFY;
+		state bool disableGet = BUGGIFY;
+		state bool disableCommit = BUGGIFY;
+		state bool disableClearRange = BUGGIFY;
+		state bool disableClear = BUGGIFY;
+		state bool disableWatch = BUGGIFY;
+		state bool disableWriteConflictRange = BUGGIFY;
+		state bool disableDelay = BUGGIFY;
+		state bool disableReset = BUGGIFY;
+		state bool disableReadConflictRange = BUGGIFY;
+		state bool disableSet = BUGGIFY;
+		state bool disableAtomicOp = BUGGIFY;
+
+		state Key timebombStr = makeString(8);
+		uint8_t* data = mutateString(timebombStr);
+		memcpy(data, &timebomb, 8);
 
 		loop {
-			wait(delay(now() - startTime > self->slowModeStart ||
-			                   (g_network->isSimulated() && g_simulator.speedUpSimulation)
-			               ? 1.0
-			               : 0.1));
-			try {
-				wait(self->randomTransaction(
-				    (self->useExtraDB && deterministicRandom()->random01() < 0.5) ? self->extraDB : cx,
-				    self,
-				    startTime));
-			} catch (Error& e) {
-				if (e.code() != error_code_not_committed)
-					throw;
-				break;
-			}
-			if (now() - startTime > self->testDuration)
+			if (now() - testStartTime > self->testDuration) {
 				return Void();
-		}
-	}
-}
+			}
 
-Key getRandomKey() {
-	return getKeyForIndex(deterministicRandom()->randomInt(0, nodes));
-}
-
-Key getKeyForIndex(int idx) {
-	idx += minNode;
-	if (adjacentKeys) {
-		return Key(idx ? keyPrefix + std::string(idx, '\x00') : "");
-	} else {
-		return Key(keyPrefix + format("%010d", idx));
-	}
-}
-
-Key versionStampKeyForIndex(int idx) {
-	Key result = KeyRef(getKeyForIndex(idx).toString() + std::string(14, '\x00'));
-	int32_t pos = deterministicRandom()->randomInt(0, result.size() - 13);
-	pos = littleEndian32(pos);
-	uint8_t* data = mutateString(result);
-	memcpy(data + result.size() - sizeof(int32_t), &pos, sizeof(int32_t));
-	return result;
-}
-
-Key getRandomVersionStampKey() {
-	return versionStampKeyForIndex(deterministicRandom()->randomInt(0, nodes));
-}
-
-KeySelector getRandomKeySelector() {
-	int scale = 1 << deterministicRandom()->randomInt(0, 14);
-	return KeySelectorRef(
-	    getRandomKey(), deterministicRandom()->random01() < 0.5, deterministicRandom()->randomInt(-scale, scale));
-}
-
-GetRangeLimits getRandomLimits() {
-	int kind = deterministicRandom()->randomInt(0, 3);
-	return GetRangeLimits(
-	    (kind & 1) ? GetRangeLimits::ROW_LIMIT_UNLIMITED
-	               : deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(1, 10)),
-	    (kind & 2) ? GetRangeLimits::BYTE_LIMIT_UNLIMITED
-	               : deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(1, 15)));
-}
-
-KeyRange getRandomRange(int sizeLimit) {
-	int startLocation = deterministicRandom()->randomInt(0, nodes);
-	int scale = deterministicRandom()->randomInt(
-	    0, deterministicRandom()->randomInt(2, 5) * deterministicRandom()->randomInt(2, 5));
-	int endLocation = startLocation + deterministicRandom()->randomInt(
-	                                      0, 1 + std::min(sizeLimit, std::min(nodes - startLocation, 1 << scale)));
-
-	return KeyRangeRef(getKeyForIndex(startLocation), getKeyForIndex(endLocation));
-}
-
-Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type) {
-	Arena arena;
-	if (type == MutationRef::SetValue)
-		return value;
-	else if (type == MutationRef::AddValue)
-		return doLittleEndianAdd(existingValue, value, arena);
-	else if (type == MutationRef::AppendIfFits)
-		return doAppendIfFits(existingValue, value, arena);
-	else if (type == MutationRef::And)
-		return doAndV2(existingValue, value, arena);
-	else if (type == MutationRef::Or)
-		return doOr(existingValue, value, arena);
-	else if (type == MutationRef::Xor)
-		return doXor(existingValue, value, arena);
-	else if (type == MutationRef::Max)
-		return doMax(existingValue, value, arena);
-	else if (type == MutationRef::Min)
-		return doMinV2(existingValue, value, arena);
-	else if (type == MutationRef::ByteMin)
-		return doByteMin(existingValue, value, arena);
-	else if (type == MutationRef::ByteMax)
-		return doByteMax(existingValue, value, arena);
-	ASSERT(false);
-	return Value();
-}
-
-ACTOR Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self, double testStartTime) {
-	state ReadYourWritesTransaction tr(cx);
-	state bool readYourWritesDisabled = deterministicRandom()->random01() < 0.5;
-	state bool readAheadDisabled = deterministicRandom()->random01() < 0.5;
-	state bool snapshotRYWDisabled = deterministicRandom()->random01() < 0.5;
-	state bool useBatchPriority = deterministicRandom()->random01() < 0.5;
-	state int64_t timebomb =
-	    (FLOW_KNOBS->MAX_BUGGIFIED_DELAY == 0.0 && deterministicRandom()->random01() < 0.01)
-	        ? deterministicRandom()->randomInt64(1, 6000)
-	        : 0; // timebomb check can fail incorrectly if simulation injects delay longer than the timebomb
-	state std::vector<Future<Void>> operations;
-	state ActorCollection commits(false);
-	state std::vector<Future<Void>> watches;
-	state int changeNum = 1;
-	state bool doingCommit = false;
-	state int waitLocation = 0;
-	state double startTime = now();
-
-	state bool disableGetKey = BUGGIFY;
-	state bool disableGetRange = BUGGIFY;
-	state bool disableGet = BUGGIFY;
-	state bool disableCommit = BUGGIFY;
-	state bool disableClearRange = BUGGIFY;
-	state bool disableClear = BUGGIFY;
-	state bool disableWatch = BUGGIFY;
-	state bool disableWriteConflictRange = BUGGIFY;
-	state bool disableDelay = BUGGIFY;
-	state bool disableReset = BUGGIFY;
-	state bool disableReadConflictRange = BUGGIFY;
-	state bool disableSet = BUGGIFY;
-	state bool disableAtomicOp = BUGGIFY;
-
-	state Key timebombStr = makeString(8);
-	uint8_t* data = mutateString(timebombStr);
-	memcpy(data, &timebomb, 8);
-
-	loop {
-		if (now() - testStartTime > self->testDuration) {
-			return Void();
-		}
-
-		state int64_t memLimit = 1e8;
-		state bool cancelled = false;
-		if (readYourWritesDisabled)
-			tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
-		if (snapshotRYWDisabled)
-			tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
-		if (readAheadDisabled)
-			tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
-		if (useBatchPriority)
-			tr.setOption(FDBTransactionOptions::PRIORITY_BATCH);
-		if (self->useSystemKeys)
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
-		tr.addWriteConflictRange(self->conflictRange);
-		self->addedConflicts.insert(self->conflictRange, true);
-		try {
-			state int numWaits = deterministicRandom()->randomInt(1, 5);
-			state int i = 0;
-			for (; i < numWaits && memLimit > 0; i++) {
-				//TraceEvent("WDROps").detail("Count", i).detail("Max", numWaits).detail("ReadYourWritesDisabled",readYourWritesDisabled);
-				state int numOps = deterministicRandom()->randomInt(1, self->numOps);
-				state int j = 0;
-				for (; j < numOps && memLimit > 0; j++) {
-					if (commits.getResult().isError())
-						throw commits.getResult().getError();
-					try {
-						state int operationType = deterministicRandom()->randomInt(0, 21);
-						if (operationType == 0 && !disableGetKey) {
-							operations.push_back(self->getKeyAndCompare(&tr,
-							                                            self->getRandomKeySelector(),
-							                                            Snapshot{ deterministicRandom()->coinflip() },
-							                                            readYourWritesDisabled,
-							                                            snapshotRYWDisabled,
-							                                            self,
-							                                            &doingCommit,
-							                                            &memLimit));
-						} else if (operationType == 1 && !disableGetRange) {
-							operations.push_back(self->getRangeAndCompare(&tr,
-							                                              self->getRandomKeySelector(),
-							                                              self->getRandomKeySelector(),
-							                                              self->getRandomLimits(),
-							                                              Snapshot{ deterministicRandom()->coinflip() },
-							                                              Reverse{ deterministicRandom()->coinflip() },
-							                                              readYourWritesDisabled,
-							                                              snapshotRYWDisabled,
-							                                              self,
-							                                              &doingCommit,
-							                                              &memLimit));
-						} else if (operationType == 2 && !disableGet) {
-							operations.push_back(self->getAndCompare(&tr,
-							                                         self->getRandomKey(),
-							                                         Snapshot{ deterministicRandom()->coinflip() },
-							                                         readYourWritesDisabled,
-							                                         snapshotRYWDisabled,
-							                                         self,
-							                                         &doingCommit,
-							                                         &memLimit));
-						} else if (operationType == 3 && !disableCommit) {
-							if (!self->rarelyCommit || deterministicRandom()->random01() < 1.0 / self->numOps) {
-								Future<Void> commit = self->commitAndUpdateMemory(&tr,
-								                                                  self,
-								                                                  &cancelled,
-								                                                  readYourWritesDisabled,
-								                                                  snapshotRYWDisabled,
-								                                                  readAheadDisabled,
-								                                                  useBatchPriority,
-								                                                  &doingCommit,
-								                                                  &startTime,
-								                                                  timebombStr);
-								operations.push_back(commit);
-								commits.add(commit);
-							}
-						} else if (operationType == 4 && !disableClearRange) {
-							KeyRange range = self->getRandomRange(self->maxClearSize);
-							self->changeCount.insert(range, changeNum++);
-							bool noConflict = deterministicRandom()->random01() < 0.5;
-							//TraceEvent("WDRClearRange").detail("Begin", printable(range)).detail("NoConflict", noConflict);
-							if (noConflict)
-								tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-							tr.clear(range);
-							if (!noConflict) {
-								KeyRangeRef conflict(
-								    range.begin.substr(0,
-								                       std::min<int>(range.begin.size(),
-								                                     (range.begin.startsWith(systemKeys.begin)
-								                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-								                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-								                                         1)),
-								    range.end.substr(0,
-								                     std::min<int>(range.end.size(),
-								                                   (range.end.startsWith(systemKeys.begin)
-								                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-								                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-								                                       1)));
-								self->addedConflicts.insert(conflict, true);
-							}
-							self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
-							                           self->memoryDatabase.lower_bound(range.end));
-						} else if (operationType == 5 && !disableClear) {
-							Key key = self->getRandomKey();
-							self->changeCount.insert(key, changeNum++);
-							bool noConflict = deterministicRandom()->random01() < 0.5;
-							//TraceEvent("WDRClear").detail("Key", printable(key)).detail("NoConflict", noConflict);
-							if (noConflict)
-								tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-							tr.clear(key);
-							if (!noConflict &&
-							    key.size() <= (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-							                                                    : CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
-								self->addedConflicts.insert(key, true);
-							}
-							self->memoryDatabase.erase(key);
-						} else if (operationType == 6 && !disableWatch) {
-							watches.push_back(self->watchAndCompare(
-							    &tr, self->getRandomKey(), readYourWritesDisabled, self, &doingCommit, &memLimit));
-						} else if (operationType == 7 && !disableWriteConflictRange) {
-							KeyRange range = self->getRandomRange(self->nodes);
-							//TraceEvent("WDRAddWriteConflict").detail("Range", printable(range));
-							tr.addWriteConflictRange(range);
-							KeyRangeRef conflict(
-							    range.begin.substr(0,
-							                       std::min<int>(range.begin.size(),
-							                                     (range.begin.startsWith(systemKeys.begin)
-							                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-							                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-							                                         1)),
-							    range.end.substr(0,
-							                     std::min<int>(range.end.size(),
-							                                   (range.end.startsWith(systemKeys.begin)
-							                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-							                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-							                                       1)));
-							self->addedConflicts.insert(conflict, true);
-						} else if (operationType == 8 && !disableDelay) {
-							double maxTime = 6.0;
-							if (timebomb > 0)
-								maxTime = startTime + timebomb / 1000.0 - now();
-							operations.push_back(
-							    delay(deterministicRandom()->random01() * deterministicRandom()->random01() *
-							          deterministicRandom()->random01() * maxTime));
-						} else if (operationType == 9 && !disableReset) {
-							if (deterministicRandom()->random01() < 0.001) {
-								//TraceEvent("WDRReset");
-								tr.reset();
-								self->memoryDatabase = self->lastCommittedDatabase;
-								self->addedConflicts.insert(allKeys, false);
-								if (readYourWritesDisabled)
-									tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
-								if (snapshotRYWDisabled)
-									tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
-								if (readAheadDisabled)
-									tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
-								if (self->useSystemKeys)
-									tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-								tr.addWriteConflictRange(self->conflictRange);
-								self->addedConflicts.insert(self->conflictRange, true);
-								startTime = now();
-								tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
-							}
-						} else if (operationType == 10 && !disableReadConflictRange) {
-							KeyRange range = self->getRandomRange(self->maxClearSize);
-							tr.addReadConflictRange(range);
-						} else if (operationType == 11 && !disableAtomicOp) {
-							if (!self->useSystemKeys && deterministicRandom()->random01() < 0.01) {
-								Key versionStampKey = self->getRandomVersionStampKey();
-								Value value = self->getRandomValue();
-								KeyRangeRef range = getVersionstampKeyRange(versionStampKey.arena(),
-								                                            versionStampKey,
-								                                            tr.getCachedReadVersion().orDefault(0),
-								                                            normalKeys.end);
-								self->changeCount.insert(range, changeNum++);
-								//TraceEvent("WDRVersionStamp").detail("VersionStampKey", printable(versionStampKey)).detail("Range", printable(range));
-								tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
-								tr.clear(range);
-								KeyRangeRef conflict(
-								    range.begin.substr(0,
-								                       std::min<int>(range.begin.size(),
-								                                     (range.begin.startsWith(systemKeys.begin)
-								                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-								                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-								                                         1)),
-								    range.end.substr(0,
-								                     std::min<int>(range.end.size(),
-								                                   (range.end.startsWith(systemKeys.begin)
-								                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-								                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-								                                       1)));
-								self->addedConflicts.insert(conflict, true);
-								self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
-								                           self->memoryDatabase.lower_bound(range.end));
-							} else {
-								Key key = self->getRandomKey();
-								Value value = self->getRandomValue();
-								MutationRef::Type opType;
-								switch (deterministicRandom()->randomInt(0, 8)) {
-								case 0:
-									opType = MutationRef::AddValue;
-									break;
-								case 1:
-									opType = MutationRef::And;
-									break;
-								case 2:
-									opType = MutationRef::Or;
-									break;
-								case 3:
-									opType = MutationRef::Xor;
-									break;
-								case 4:
-									opType = MutationRef::Max;
-									break;
-								case 5:
-									opType = MutationRef::Min;
-									break;
-								case 6:
-									opType = MutationRef::ByteMin;
-									break;
-								case 7:
-									opType = MutationRef::ByteMax;
-									break;
+			state int64_t memLimit = 1e8;
+			state bool cancelled = false;
+			if (readYourWritesDisabled)
+				tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+			if (snapshotRYWDisabled)
+				tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
+			if (readAheadDisabled)
+				tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
+			if (useBatchPriority)
+				tr.setOption(FDBTransactionOptions::PRIORITY_BATCH);
+			if (self->useSystemKeys)
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
+			tr.addWriteConflictRange(self->conflictRange);
+			self->addedConflicts.insert(self->conflictRange, true);
+			try {
+				state int numWaits = deterministicRandom()->randomInt(1, 5);
+				state int i = 0;
+				for (; i < numWaits && memLimit > 0; i++) {
+					//TraceEvent("WDROps").detail("Count", i).detail("Max", numWaits).detail("ReadYourWritesDisabled",readYourWritesDisabled);
+					state int numOps = deterministicRandom()->randomInt(1, self->numOps);
+					state int j = 0;
+					for (; j < numOps && memLimit > 0; j++) {
+						if (commits.getResult().isError())
+							throw commits.getResult().getError();
+						try {
+							state int operationType = deterministicRandom()->randomInt(0, 21);
+							if (operationType == 0 && !disableGetKey) {
+								operations.push_back(
+								    self->getKeyAndCompare(&tr,
+								                           self->getRandomKeySelector(),
+								                           Snapshot{ deterministicRandom()->coinflip() },
+								                           readYourWritesDisabled,
+								                           snapshotRYWDisabled,
+								                           self,
+								                           &doingCommit,
+								                           &memLimit));
+							} else if (operationType == 1 && !disableGetRange) {
+								operations.push_back(
+								    self->getRangeAndCompare(&tr,
+								                             self->getRandomKeySelector(),
+								                             self->getRandomKeySelector(),
+								                             self->getRandomLimits(),
+								                             Snapshot{ deterministicRandom()->coinflip() },
+								                             Reverse{ deterministicRandom()->coinflip() },
+								                             readYourWritesDisabled,
+								                             snapshotRYWDisabled,
+								                             self,
+								                             &doingCommit,
+								                             &memLimit));
+							} else if (operationType == 2 && !disableGet) {
+								operations.push_back(self->getAndCompare(&tr,
+								                                         self->getRandomKey(),
+								                                         Snapshot{ deterministicRandom()->coinflip() },
+								                                         readYourWritesDisabled,
+								                                         snapshotRYWDisabled,
+								                                         self,
+								                                         &doingCommit,
+								                                         &memLimit));
+							} else if (operationType == 3 && !disableCommit) {
+								if (!self->rarelyCommit || deterministicRandom()->random01() < 1.0 / self->numOps) {
+									Future<Void> commit = self->commitAndUpdateMemory(&tr,
+									                                                  self,
+									                                                  &cancelled,
+									                                                  readYourWritesDisabled,
+									                                                  snapshotRYWDisabled,
+									                                                  readAheadDisabled,
+									                                                  useBatchPriority,
+									                                                  &doingCommit,
+									                                                  &startTime,
+									                                                  timebombStr);
+									operations.push_back(commit);
+									commits.add(commit);
 								}
-								self->changeCount.insert(key, changeNum++);
+							} else if (operationType == 4 && !disableClearRange) {
+								KeyRange range = self->getRandomRange(self->maxClearSize);
+								self->changeCount.insert(range, changeNum++);
 								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRAtomicOp").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
+								//TraceEvent("WDRClearRange").detail("Begin", printable(range)).detail("NoConflict", noConflict);
 								if (noConflict)
 									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-								tr.atomicOp(key, value, opType);
-								//TraceEvent("WDRAtomicOpSuccess").detail("Key", printable(key)).detail("Value", value.size());
+								tr.clear(range);
+								if (!noConflict) {
+									KeyRangeRef conflict(
+									    range.begin.substr(0,
+									                       std::min<int>(range.begin.size(),
+									                                     (range.begin.startsWith(systemKeys.begin)
+									                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+									                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+									                                         1)),
+									    range.end.substr(0,
+									                     std::min<int>(range.end.size(),
+									                                   (range.end.startsWith(systemKeys.begin)
+									                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+									                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+									                                       1)));
+									self->addedConflicts.insert(conflict, true);
+								}
+								self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
+								                           self->memoryDatabase.lower_bound(range.end));
+							} else if (operationType == 5 && !disableClear) {
+								Key key = self->getRandomKey();
+								self->changeCount.insert(key, changeNum++);
+								bool noConflict = deterministicRandom()->random01() < 0.5;
+								//TraceEvent("WDRClear").detail("Key", printable(key)).detail("NoConflict", noConflict);
+								if (noConflict)
+									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
+								tr.clear(key);
+								if (!noConflict && key.size() <= (key.startsWith(systemKeys.begin)
+								                                      ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+								                                      : CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
+									self->addedConflicts.insert(key, true);
+								}
+								self->memoryDatabase.erase(key);
+							} else if (operationType == 6 && !disableWatch) {
+								watches.push_back(self->watchAndCompare(
+								    &tr, self->getRandomKey(), readYourWritesDisabled, self, &doingCommit, &memLimit));
+							} else if (operationType == 7 && !disableWriteConflictRange) {
+								KeyRange range = self->getRandomRange(self->nodes);
+								//TraceEvent("WDRAddWriteConflict").detail("Range", printable(range));
+								tr.addWriteConflictRange(range);
+								KeyRangeRef conflict(
+								    range.begin.substr(0,
+								                       std::min<int>(range.begin.size(),
+								                                     (range.begin.startsWith(systemKeys.begin)
+								                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+								                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+								                                         1)),
+								    range.end.substr(0,
+								                     std::min<int>(range.end.size(),
+								                                   (range.end.startsWith(systemKeys.begin)
+								                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+								                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+								                                       1)));
+								self->addedConflicts.insert(conflict, true);
+							} else if (operationType == 8 && !disableDelay) {
+								double maxTime = 6.0;
+								if (timebomb > 0)
+									maxTime = startTime + timebomb / 1000.0 - now();
+								operations.push_back(
+								    delay(deterministicRandom()->random01() * deterministicRandom()->random01() *
+								          deterministicRandom()->random01() * maxTime));
+							} else if (operationType == 9 && !disableReset) {
+								if (deterministicRandom()->random01() < 0.001) {
+									//TraceEvent("WDRReset");
+									tr.reset();
+									self->memoryDatabase = self->lastCommittedDatabase;
+									self->addedConflicts.insert(allKeys, false);
+									if (readYourWritesDisabled)
+										tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+									if (snapshotRYWDisabled)
+										tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
+									if (readAheadDisabled)
+										tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
+									if (self->useSystemKeys)
+										tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+									tr.addWriteConflictRange(self->conflictRange);
+									self->addedConflicts.insert(self->conflictRange, true);
+									startTime = now();
+									tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
+								}
+							} else if (operationType == 10 && !disableReadConflictRange) {
+								KeyRange range = self->getRandomRange(self->maxClearSize);
+								tr.addReadConflictRange(range);
+							} else if (operationType == 11 && !disableAtomicOp) {
+								if (!self->useSystemKeys && deterministicRandom()->random01() < 0.01) {
+									Key versionStampKey = self->getRandomVersionStampKey();
+									Value value = self->getRandomValue();
+									KeyRangeRef range = getVersionstampKeyRange(versionStampKey.arena(),
+									                                            versionStampKey,
+									                                            tr.getCachedReadVersion().orDefault(0),
+									                                            normalKeys.end);
+									self->changeCount.insert(range, changeNum++);
+									//TraceEvent("WDRVersionStamp").detail("VersionStampKey", printable(versionStampKey)).detail("Range", printable(range));
+									tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
+									tr.clear(range);
+									KeyRangeRef conflict(
+									    range.begin.substr(0,
+									                       std::min<int>(range.begin.size(),
+									                                     (range.begin.startsWith(systemKeys.begin)
+									                                          ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+									                                          : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+									                                         1)),
+									    range.end.substr(0,
+									                     std::min<int>(range.end.size(),
+									                                   (range.end.startsWith(systemKeys.begin)
+									                                        ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+									                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+									                                       1)));
+									self->addedConflicts.insert(conflict, true);
+									self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
+									                           self->memoryDatabase.lower_bound(range.end));
+								} else {
+									Key key = self->getRandomKey();
+									Value value = self->getRandomValue();
+									MutationRef::Type opType;
+									switch (deterministicRandom()->randomInt(0, 8)) {
+									case 0:
+										opType = MutationRef::AddValue;
+										break;
+									case 1:
+										opType = MutationRef::And;
+										break;
+									case 2:
+										opType = MutationRef::Or;
+										break;
+									case 3:
+										opType = MutationRef::Xor;
+										break;
+									case 4:
+										opType = MutationRef::Max;
+										break;
+									case 5:
+										opType = MutationRef::Min;
+										break;
+									case 6:
+										opType = MutationRef::ByteMin;
+										break;
+									case 7:
+										opType = MutationRef::ByteMax;
+										break;
+									}
+									self->changeCount.insert(key, changeNum++);
+									bool noConflict = deterministicRandom()->random01() < 0.5;
+									//TraceEvent("WDRAtomicOp").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
+									if (noConflict)
+										tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
+									tr.atomicOp(key, value, opType);
+									//TraceEvent("WDRAtomicOpSuccess").detail("Key", printable(key)).detail("Value", value.size());
+									if (!noConflict && key.size() <= (key.startsWith(systemKeys.begin)
+									                                      ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+									                                      : CLIENT_KNOBS->KEY_SIZE_LIMIT))
+										self->addedConflicts.insert(key, true);
+									Optional<Value> existing = self->memoryGet(&self->memoryDatabase, key);
+									self->memoryDatabase[key] =
+									    self->applyAtomicOp(existing.present() ? Optional<StringRef>(existing.get())
+									                                           : Optional<StringRef>(),
+									                        value,
+									                        opType);
+								}
+							} else if (operationType > 11 && !disableSet) {
+								Key key = self->getRandomKey();
+								Value value = self->getRandomValue();
+								self->changeCount.insert(key, changeNum++);
+								bool noConflict = deterministicRandom()->random01() < 0.5;
+								//TraceEvent("WDRSet").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
+								if (noConflict)
+									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
+								tr.set(key, value);
 								if (!noConflict && key.size() <= (key.startsWith(systemKeys.begin)
 								                                      ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
 								                                      : CLIENT_KNOBS->KEY_SIZE_LIMIT))
 									self->addedConflicts.insert(key, true);
-								Optional<Value> existing = self->memoryGet(&self->memoryDatabase, key);
-								self->memoryDatabase[key] = self->applyAtomicOp(
-								    existing.present() ? Optional<StringRef>(existing.get()) : Optional<StringRef>(),
-								    value,
-								    opType);
+								//TraceEvent("WDRSetSuccess").detail("Key", printable(key)).detail("Value", value.size());
+								self->memoryDatabase[key] = value;
+							}
+						} catch (Error& e) {
+							if (e.code() == error_code_used_during_commit) {
+								ASSERT(doingCommit);
+							} else if (e.code() != error_code_transaction_cancelled) {
+								throw;
 							}
-						} else if (operationType > 11 && !disableSet) {
-							Key key = self->getRandomKey();
-							Value value = self->getRandomValue();
-							self->changeCount.insert(key, changeNum++);
-							bool noConflict = deterministicRandom()->random01() < 0.5;
-							//TraceEvent("WDRSet").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
-							if (noConflict)
-								tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-							tr.set(key, value);
-							if (!noConflict &&
-							    key.size() <= (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-							                                                    : CLIENT_KNOBS->KEY_SIZE_LIMIT))
-								self->addedConflicts.insert(key, true);
-							//TraceEvent("WDRSetSuccess").detail("Key", printable(key)).detail("Value", value.size());
-							self->memoryDatabase[key] = value;
-						}
-					} catch (Error& e) {
-						if (e.code() == error_code_used_during_commit) {
-							ASSERT(doingCommit);
-						} else if (e.code() != error_code_transaction_cancelled) {
-							throw;
 						}
 					}
-				}
 
-				if (waitLocation < operations.size()) {
-					int waitOp = deterministicRandom()->randomInt(waitLocation, operations.size());
-					//TraceEvent("WDRWait").detail("Op", waitOp).detail("Operations", operations.size()).detail("WaitLocation", waitLocation);
-					wait(operations[waitOp]);
-					wait(delay(0.000001)); // to ensure errors have propgated from reads to commits
-					waitLocation = operations.size();
+					if (waitLocation < operations.size()) {
+						int waitOp = deterministicRandom()->randomInt(waitLocation, operations.size());
+						//TraceEvent("WDRWait").detail("Op", waitOp).detail("Operations", operations.size()).detail("WaitLocation", waitLocation);
+						wait(operations[waitOp]);
+						wait(delay(0.000001)); // to ensure errors have propgated from reads to commits
+						waitLocation = operations.size();
+					}
+				}
+				wait(waitForAll(operations));
+				ASSERT(timebomb == 0 || 1000 * (now() - startTime) <= timebomb + 1);
+				wait(tr.debug_onIdle());
+				wait(delay(0.000001)); // to ensure triggered watches have a change to register
+				self->finished.trigger();
+				wait(waitForAll(watches)); // only for errors, should have all returned
+				self->changeCount.insert(allKeys, 0);
+				break;
+			} catch (Error& e) {
+				operations.clear();
+				commits.clear(false);
+				waitLocation = 0;
+				watches.clear();
+				self->changeCount.insert(allKeys, 0);
+				doingCommit = false;
+				//TraceEvent("WDRError").errorUnsuppressed(e);
+				if (e.code() == error_code_database_locked) {
+					self->memoryDatabase = self->lastCommittedDatabase;
+					self->addedConflicts.insert(allKeys, false);
+					return Void();
+				}
+				if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result ||
+				    e.code() == error_code_transaction_too_large || e.code() == error_code_key_too_large ||
+				    e.code() == error_code_value_too_large || e.code() == error_code_too_many_watches || cancelled)
+					throw not_committed();
+				try {
+					wait(tr.onError(e));
+				} catch (Error& e) {
+					if (e.code() == error_code_transaction_timed_out) {
+						ASSERT(timebomb != 0 && 1000 * (now() - startTime) >= timebomb - 1);
+						throw not_committed();
+					}
+					throw e;
 				}
-			}
-			wait(waitForAll(operations));
-			ASSERT(timebomb == 0 || 1000 * (now() - startTime) <= timebomb + 1);
-			wait(tr.debug_onIdle());
-			wait(delay(0.000001)); // to ensure triggered watches have a change to register
-			self->finished.trigger();
-			wait(waitForAll(watches)); // only for errors, should have all returned
-			self->changeCount.insert(allKeys, 0);
-			break;
-		} catch (Error& e) {
-			operations.clear();
-			commits.clear(false);
-			waitLocation = 0;
-			watches.clear();
-			self->changeCount.insert(allKeys, 0);
-			doingCommit = false;
-			//TraceEvent("WDRError").errorUnsuppressed(e);
-			if (e.code() == error_code_database_locked) {
 				self->memoryDatabase = self->lastCommittedDatabase;
 				self->addedConflicts.insert(allKeys, false);
-				return Void();
 			}
-			if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result ||
-			    e.code() == error_code_transaction_too_large || e.code() == error_code_key_too_large ||
-			    e.code() == error_code_value_too_large || e.code() == error_code_too_many_watches || cancelled)
-				throw not_committed();
-			try {
-				wait(tr.onError(e));
-			} catch (Error& e) {
-				if (e.code() == error_code_transaction_timed_out) {
-					ASSERT(timebomb != 0 && 1000 * (now() - startTime) >= timebomb - 1);
-					throw not_committed();
-				}
-				throw e;
-			}
-			self->memoryDatabase = self->lastCommittedDatabase;
-			self->addedConflicts.insert(allKeys, false);
 		}
+		self->memoryDatabase = self->lastCommittedDatabase;
+		self->addedConflicts.insert(allKeys, false);
+		return Void();
 	}
-	self->memoryDatabase = self->lastCommittedDatabase;
-	self->addedConflicts.insert(allKeys, false);
-	return Void();
-}
-}
-;
+};
 
 WorkloadFactory<WriteDuringReadWorkload> WriteDuringReadWorkloadFactory("WriteDuringRead");

From f6b24cd6440779a1ddc846946efa49ffc48c7e5f Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 1 Mar 2022 09:04:15 -0800
Subject: [PATCH 137/210] Add some extra print debugging if we overflow the
 trace buffer.

---
 flow/Trace.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index d8a04273ea..0afbc60c85 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -399,6 +399,8 @@ public:
 		// Without this we would never see any trace events from that loop, and it would be more difficult to identify
 		// where the process is actually stuck.
 		if (g_network && g_network->isSimulated() && bufferLength > 1e8) {
+			fprintf(stderr, "Trace log buffer overflow\n");
+			fprintf(stderr, "Last event: %s\n", fields.toString().c_str());
 			// Setting this to 0 avoids a recurse from the assertion trace event and also prevents a situation where
 			// we roll the trace log only to log the single assertion event when using --crash.
 			bufferLength = 0;

From beac368ba95d7a33e745c338f1f43096a1d1ff32 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Wed, 23 Feb 2022 08:13:10 -0800
Subject: [PATCH 138/210] Add a mechanism to enable error backtrace logging for
 select errors

---
 flow/Error.cpp | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/flow/Error.cpp b/flow/Error.cpp
index 64f8c7d60c..16776661c5 100644
--- a/flow/Error.cpp
+++ b/flow/Error.cpp
@@ -27,6 +27,13 @@
 
 bool g_crashOnError = false;
 
+#define DEBUG_ERROR 0
+
+#if DEBUG_ERROR
+std::set<int> debugErrorSet = std::set<int>{ error_code_platform_error };
+#define SHOULD_LOG_ERROR(x) (debugErrorSet.count(x) > 0)
+#endif
+
 #include <iostream>
 
 Error Error::fromUnvalidatedCode(int code) {
@@ -122,6 +129,24 @@ Error::Error(int error_code) : error_code(error_code), flags(0) {
 			crashAndDie();
 		}
 	}
+
+#if DEBUG_ERROR
+	if (SHOULD_LOG_ERROR(error_code)) {
+		TraceEvent te(SevWarn, "DebugError");
+		te.error(*this).backtrace();
+		if (error_code == error_code_unknown_error) {
+			auto exception = std::current_exception();
+			if (exception) {
+				try {
+					std::rethrow_exception(exception);
+				} catch (std::exception& e) {
+					te.detail("StdException", e.what());
+				} catch (...) {
+				}
+			}
+		}
+	}
+#endif
 }
 
 ErrorCodeTable& Error::errorCodeTable() {

From 92a888910f61110e1066d5f4d61157181fc5362b Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Tue, 1 Mar 2022 08:20:46 -0600
Subject: [PATCH 139/210] Fix many repeated recruitment bug

---
 fdbserver/DDTeamCollection.actor.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 34c661763e..bc22b1a7eb 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2245,6 +2245,11 @@ public:
 			// Ask the candidateWorker to initialize a SS only if the worker does not have a pending request
 			state UID interfaceId = deterministicRandom()->randomUniqueID();
 
+			// insert recruiting localities BEFORE actor waits, to ensure we don't send many recruitment requests to the
+			// same storage
+			self->recruitingIds.insert(interfaceId);
+			self->recruitingLocalities.insert(candidateWorker.worker.stableAddress());
+
 			UID clusterId = wait(self->getClusterId());
 
 			state InitializeStorageRequest isr;
@@ -2255,9 +2260,6 @@ public:
 			isr.interfaceId = interfaceId;
 			isr.clusterId = clusterId;
 
-			self->recruitingIds.insert(interfaceId);
-			self->recruitingLocalities.insert(candidateWorker.worker.stableAddress());
-
 			// if tss, wait for pair ss to finish and add its id to isr. If pair fails, don't recruit tss
 			state bool doRecruit = true;
 			if (recruitTss) {

From 9a45df8e6cce66c103a55500f4ba8f3dca52ec39 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Tue, 1 Mar 2022 18:08:50 -0800
Subject: [PATCH 140/210] Change fdb_c's dep on fdbclient to be private (#6466)

That way transitive dependencies (e.g. libfdb_java.so) don't include
unnecessary fdbclient code. Related: #6455
---
 bindings/c/CMakeLists.txt    | 14 +++++++-------
 bindings/flow/CMakeLists.txt |  1 +
 bindings/java/CMakeLists.txt |  1 +
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 8ba8e217fc..78dce4cdc9 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -51,7 +51,7 @@ endif()
 add_dependencies(fdb_c fdb_c_generated fdb_c_options)
 add_dependencies(fdbclient fdb_c_options)
 add_dependencies(fdbclient_sampling fdb_c_options)
-target_link_libraries(fdb_c PUBLIC $<BUILD_INTERFACE:fdbclient>)
+target_link_libraries(fdb_c PRIVATE $<BUILD_INTERFACE:fdbclient>)
 if(APPLE)
   set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
   add_custom_command(OUTPUT ${symbols}
@@ -123,9 +123,9 @@ if(NOT WIN32 AND NOT IS_ARM_MAC)
     strip_debug_symbols(fdb_c_ryw_benchmark)
     strip_debug_symbols(fdb_c_txn_size_test)
   endif()
-  target_link_libraries(fdb_c_performance_test PRIVATE fdb_c)
-  target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c)
-  target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c)
+  target_link_libraries(fdb_c_performance_test PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads)
 
   add_dependencies(fdb_c_setup_tests doctest)
   add_dependencies(fdb_c_unit_tests doctest)
@@ -136,14 +136,14 @@ if(NOT WIN32 AND NOT IS_ARM_MAC)
   target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
   target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
   target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
-  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient)
   target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
-  target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)
   target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
 
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
-  target_link_libraries(mako PRIVATE fdb_c)
+  target_link_libraries(mako PRIVATE fdb_c fdbclient)
 
   if(NOT OPEN_FOR_IDE)
     # Make sure that fdb_c.h is compatible with c90
diff --git a/bindings/flow/CMakeLists.txt b/bindings/flow/CMakeLists.txt
index 29d5d8fcb9..34dd13e051 100644
--- a/bindings/flow/CMakeLists.txt
+++ b/bindings/flow/CMakeLists.txt
@@ -18,6 +18,7 @@ set(SRCS
 
 add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS})
 target_link_libraries(fdb_flow PUBLIC fdb_c)
+target_link_libraries(fdb_flow PUBLIC fdbclient)
 target_include_directories(fdb_flow PUBLIC
   "${CMAKE_CURRENT_BINARY_DIR}"
   "${CMAKE_CURRENT_SOURCE_DIR}"
diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt
index d83305def4..9dfb349040 100644
--- a/bindings/java/CMakeLists.txt
+++ b/bindings/java/CMakeLists.txt
@@ -154,6 +154,7 @@ endif()
 set_target_properties(java_workloads PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/share/foundationdb")
 target_link_libraries(java_workloads PUBLIC fdb_c ${JNI_LIBRARIES})
+target_link_libraries(java_workloads PRIVATE flow) # mostly for boost
 target_include_directories(java_workloads PUBLIC ${JNI_INCLUDE_DIRS})
 
 set(CMAKE_JAVA_COMPILE_FLAGS "-source" "1.8" "-target" "1.8" "-XDignore.symbol.file")

From e73c0a31e6640d44fc7daa3c8c22a9d02ff3cbdb Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Wed, 2 Mar 2022 10:03:23 -0800
Subject: [PATCH 141/210] add wiggle_server_ids and wiggle_server_addresses in
 status json

---
 fdbcli/StatusCommand.actor.cpp               |  1 +
 fdbclient/NativeAPI.actor.cpp                | 26 ++++++++++++++++++
 fdbclient/NativeAPI.actor.h                  |  4 +++
 fdbclient/Schemas.cpp                        |  2 ++
 fdbserver/DDTeamCollection.actor.cpp         | 21 ++------------
 fdbserver/Status.actor.cpp                   | 29 ++++++++++++++++++--
 fdbserver/workloads/StatusWorkload.actor.cpp |  3 +-
 7 files changed, 64 insertions(+), 22 deletions(-)

diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp
index ea6bcb5293..e0b368ebe8 100644
--- a/fdbcli/StatusCommand.actor.cpp
+++ b/fdbcli/StatusCommand.actor.cpp
@@ -810,6 +810,7 @@ void printStatus(StatusObjectReader statusObj,
 				outputString = outputStringCache;
 				outputString += "\n  Unable to retrieve data status";
 			}
+			// Storage Wiggle section
 
 			// Operating space section
 			outputString += "\n\nOperating space:";
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 90d9e85cfd..4467fd64d0 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6783,6 +6783,32 @@ ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, LockAware
 	return Void();
 }
 
+ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
+                                                                                      bool primary,
+                                                                                      bool use_system_priority) {
+	state const Key readKey = perpetualStorageWiggleIDPrefix.withSuffix(primary ? "primary/"_sr : "remote/"_sr);
+	state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(readKey,
+	                                                                                          IncludeVersion());
+	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	state std::vector<std::pair<UID, StorageWiggleValue>> res;
+	// read the wiggling pairs
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			if (use_system_priority) {
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			}
+			wait(store(res, metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY)));
+			wait(tr->commit());
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+	return res;
+}
+
 ACTOR Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(Database cx,
                                                                 KeyRange keys,
                                                                 StorageMetrics limit,
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index a3fb18574a..505bbefd3a 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -480,5 +480,9 @@ inline uint64_t getWriteOperationCost(uint64_t bytes) {
 // will be 1. Otherwise, the value will be 0.
 ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::False);
 
+ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
+                                                                                      bool primary,
+                                                                                      bool use_system_priority);
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 2e9fcc53a6..c5e2f32792 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -25,6 +25,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
 {
    "cluster":{
        "storage_wiggler": {
+		 "wiggle_server_ids":["0ccb4e0feddb55"],
+		 "wiggle_server_addresses": ["127.0.0.1"],
          "primary": {
           	"last_round_start_datetime": "Wed Feb  4 09:36:37 2022 +0000",
 			"last_round_start_timestamp": 63811229797,
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index bc22b1a7eb..75747c9c65 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2828,25 +2828,8 @@ public:
 
 	// read the current map of `perpetualStorageWiggleIDPrefix`, then restore wigglingId.
 	ACTOR static Future<Void> readStorageWiggleMap(DDTeamCollection* self) {
-
-		state const Key readKey =
-		    perpetualStorageWiggleIDPrefix.withSuffix(self->primary ? "primary/"_sr : "remote/"_sr);
-		state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(readKey,
-		                                                                                          IncludeVersion());
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-		state std::vector<std::pair<UID, StorageWiggleValue>> res;
-		// read the wiggling pairs
-		loop {
-			try {
-				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-				wait(store(res, metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY)));
-				wait(tr->commit());
-				break;
-			} catch (Error& e) {
-				wait(tr->onError(e));
-			}
-		}
+		state std::vector<std::pair<UID, StorageWiggleValue>> res =
+		    wait(readStorageWiggleValues(self->cx, self->primary, false));
 		if (res.size() > 0) {
 			// SOMEDAY: support wiggle multiple SS at once
 			ASSERT(!self->wigglingId.present()); // only single process wiggle is allowed
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index c34830311d..b2da99f2f7 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -2928,6 +2928,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		state JsonBuilderObject qos;
 		state JsonBuilderObject dataOverlay;
 		state JsonBuilderObject storageWiggler;
+		state std::unordered_set<UID> wiggleServers;
 
 		statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version());
 		statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString();
@@ -3018,8 +3019,18 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			    clusterSummaryStatisticsFetcher(pMetrics, storageServerFuture, tLogFuture, &status_incomplete_reasons));
 
 			if (configuration.get().perpetualStorageWiggleSpeed > 0) {
-				wait(store(storageWiggler, storageWigglerStatsFetcher(configuration.get(), cx, true)));
-				statusObj["storage_wiggler"] = storageWiggler;
+				state Future<std::vector<std::pair<UID, StorageWiggleValue>>> primaryWiggleValues;
+				state Future<std::vector<std::pair<UID, StorageWiggleValue>>> remoteWiggleValues;
+
+				primaryWiggleValues = readStorageWiggleValues(cx, true, true);
+				remoteWiggleValues = readStorageWiggleValues(cx, false, true);
+				wait(store(storageWiggler, storageWigglerStatsFetcher(configuration.get(), cx, true)) &&
+				     success(primaryWiggleValues) && success(remoteWiggleValues));
+
+				for (auto& p : primaryWiggleValues.get())
+					wiggleServers.insert(p.first);
+				for (auto& p : remoteWiggleValues.get())
+					wiggleServers.insert(p.first);
 			}
 
 			state std::vector<JsonBuilderObject> workerStatuses = wait(getAll(futures2));
@@ -3178,13 +3189,27 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		statusObj["datacenter_lag"] = getLagObject(datacenterVersionDifference);
 
 		int activeTSSCount = 0;
+		JsonBuilderArray wiggleServerAddress;
 		for (auto& it : storageServers) {
 			if (it.first.isTss()) {
 				activeTSSCount++;
 			}
+			if (wiggleServers.count(it.first.id())) {
+				wiggleServerAddress.push_back(it.first.address().toString());
+			}
 		}
 		statusObj["active_tss_count"] = activeTSSCount;
 
+		if (!wiggleServers.empty()) {
+			JsonBuilderArray wiggleServerUID;
+			for (auto& id : wiggleServers)
+				wiggleServerUID.push_back(id.shortString());
+
+			storageWiggler["wiggle_server_ids"] = wiggleServerUID;
+			storageWiggler["wiggle_server_addresses"] = wiggleServerAddress;
+			statusObj["storage_wiggler"] = storageWiggler;
+		}
+
 		int totalDegraded = 0;
 		for (auto& it : workers) {
 			if (it.degraded) {
diff --git a/fdbserver/workloads/StatusWorkload.actor.cpp b/fdbserver/workloads/StatusWorkload.actor.cpp
index 8d7cc36490..afd8b5c65e 100644
--- a/fdbserver/workloads/StatusWorkload.actor.cpp
+++ b/fdbserver/workloads/StatusWorkload.actor.cpp
@@ -88,7 +88,8 @@ struct StatusWorkload : TestWorkload {
 				schemaCoverage(spath, false);
 
 				if (skv.second.type() == json_spirit::array_type && skv.second.get_array().size()) {
-					schemaCoverageRequirements(skv.second.get_array()[0].get_obj(), spath + "[0]");
+					if (skv.second.get_array()[0].type() != json_spirit::str_type)
+						schemaCoverageRequirements(skv.second.get_array()[0].get_obj(), spath + "[0]");
 				} else if (skv.second.type() == json_spirit::obj_type) {
 					if (skv.second.get_obj().count("$enum")) {
 						for (auto& enum_item : skv.second.get_obj().at("$enum").get_array())

From 887d5a25cb3ec9bac6a4cd196b48bb76a5c9f087 Mon Sep 17 00:00:00 2001
From: Xiaoxi Wang <xiaoxi.wang@snowflake.com>
Date: Wed, 2 Mar 2022 11:31:55 -0800
Subject: [PATCH 142/210] add storage wiggle to status

---
 fdbcli/StatusCommand.actor.cpp | 21 +++++++++++++++++++++
 fdbserver/Status.actor.cpp     |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp
index e0b368ebe8..d1a772e0a0 100644
--- a/fdbcli/StatusCommand.actor.cpp
+++ b/fdbcli/StatusCommand.actor.cpp
@@ -811,6 +811,27 @@ void printStatus(StatusObjectReader statusObj,
 				outputString += "\n  Unable to retrieve data status";
 			}
 			// Storage Wiggle section
+			StatusObjectReader storageWigglerObj;
+			std::string storageWigglerString;
+			try {
+				if (statusObjCluster.get("storage_wiggler", storageWigglerObj)) {
+					int size = 0;
+					if (storageWigglerObj.has("wiggle_server_addresses")) {
+						storageWigglerString += "\n  Wiggle server addresses-";
+						for (auto& v : storageWigglerObj.obj().at("wiggle_server_addresses").get_array()) {
+							storageWigglerString += " " + v.get_str();
+							size += 1;
+						}
+					}
+					storageWigglerString += "\n  Wiggle server count    - " + std::to_string(size);
+				}
+			} catch (std::runtime_error&) {
+				storageWigglerString += "\n  Unable to retrieve storage wiggler status";
+			}
+			if (storageWigglerString.size()) {
+				outputString += "\n\nStorage wiggle:";
+				outputString += storageWigglerString;
+			}
 
 			// Operating space section
 			outputString += "\n\nOperating space:";
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index b2da99f2f7..d00b48cf67 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -3200,7 +3200,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		}
 		statusObj["active_tss_count"] = activeTSSCount;
 
-		if (!wiggleServers.empty()) {
+		if (!storageWiggler.empty()) {
 			JsonBuilderArray wiggleServerUID;
 			for (auto& id : wiggleServers)
 				wiggleServerUID.push_back(id.shortString());

From 8b2b74980e445fe877218cba87a8a0e164ec243d Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Wed, 2 Mar 2022 14:57:42 -0800
Subject: [PATCH 143/210] Fix name of ProxyCommitData::commitLatencyBands
 sample

---
 fdbserver/ProxyCommitData.actor.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h
index d1f9ebccf4..7ca822cfb6 100644
--- a/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/ProxyCommitData.actor.h
@@ -113,7 +113,7 @@ struct ProxyStats {
 	                        id,
 	                        SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
 	                        SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
-	    commitLatencyBands("CommitLatencyMetrics", id, SERVER_KNOBS->STORAGE_LOGGING_DELAY),
+	    commitLatencyBands("CommitLatencyBands", id, SERVER_KNOBS->STORAGE_LOGGING_DELAY),
 	    commitBatchingEmptyMessageRatio("CommitBatchingEmptyMessageRatio",
 	                                    id,
 	                                    SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,

From 83e0368eaa3eee8308ea3b5f10149b1d8690b059 Mon Sep 17 00:00:00 2001
From: Neethu Haneesha Bingi <nbingi@apple.com>
Date: Thu, 3 Mar 2022 15:14:37 -0800
Subject: [PATCH 144/210] RocksDB increasing background threads to speedup
 compaction.

---
 fdbclient/ServerKnobs.cpp                | 3 ++-
 fdbclient/ServerKnobs.h                  | 1 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 3 +++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 7bddd42b2e..de3d6d5443 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -339,7 +339,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( REPLACE_CONTENTS_BYTES,                                1e5 );
 
 	// KeyValueStoreRocksDB
-	init( ROCKSDB_BACKGROUND_PARALLELISM,                          0 );
+	init( ROCKSDB_BACKGROUND_PARALLELISM,                          4 );
 	init( ROCKSDB_READ_PARALLELISM,                                4 );
 	// Use a smaller memtable in simulation to avoid OOMs.
 	int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024;
@@ -366,6 +366,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE,                 true );
 	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
 	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 					  0.0001 );
+	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index bb7fb03a62..cfe6c8d4d1 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -297,6 +297,7 @@ public:
 	bool ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE;
 	bool ROCKSDB_PERFCONTEXT_ENABLE; // Enable rocks perf context metrics. May cause performance overhead
 	double ROCKSDB_PERFCONTEXT_SAMPLE_RATE;
+	int ROCKSDB_MAX_SUBCOMPACTIONS;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 090256df0e..1e966b8b9b 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -194,6 +194,9 @@ rocksdb::Options getOptions() {
 	if (SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM > 0) {
 		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
 	}
+	if (SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS > 0) {
+		options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
+	}
 
 	options.statistics = rocksdb::CreateDBStatistics();
 	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);

From 0ff13c1bb6eae1a606d8415b3e568020dee1b37b Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Thu, 3 Mar 2022 18:07:29 -0800
Subject: [PATCH 145/210] Allow for mako tmp directory to already exist (#6494)

---
 bindings/c/test/mako/mako.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c
index b08a3b2366..fc69ae06b5 100644
--- a/bindings/c/test/mako/mako.c
+++ b/bindings/c/test/mako/mako.c
@@ -1351,7 +1351,7 @@ void* worker_thread(void* thread_args) {
 		char str2[1000];
 		sprintf(str2, "%s%d", TEMP_DATA_STORE, *parent_id);
 		rc = mkdir(str2, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-		if (rc < 0) {
+		if (rc < 0 && errno != EEXIST) {
 			int ec = errno;
 			fprintf(stderr, "Failed to make directory: %s because %s\n", str2, strerror(ec));
 			goto failExit;

From a632d6e43ee8829c42a076b24294db1f400a8913 Mon Sep 17 00:00:00 2001
From: Mohamed Oulmahdi <oulmahdi@doxense.com>
Date: Fri, 4 Mar 2022 09:38:05 +0100
Subject: [PATCH 146/210] Fix ignoring test exit codes

---
 tests/TestRunner/TestRunner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/TestRunner/TestRunner.py b/tests/TestRunner/TestRunner.py
index 10bc7b43c2..6316e47860 100755
--- a/tests/TestRunner/TestRunner.py
+++ b/tests/TestRunner/TestRunner.py
@@ -409,7 +409,7 @@ def run_simulation_test(basedir, options):
     if len(os.listdir(wd)) == 0:
         print("Delete {} - empty".format(wd))
         os.rmdir(wd)
-    return res
+    return res and proc.returncode == 0
 
 
 if __name__ == '__main__':

From 8796a763a5afb9d37604ff9d555ab088a48a6d08 Mon Sep 17 00:00:00 2001
From: Neethu Haneesha Bingi <nbingi@apple.com>
Date: Fri, 4 Mar 2022 09:57:07 -0800
Subject: [PATCH 147/210] Rocksdb knobs for compaction, storageserver
 canCommit() waiting if rocksdb overloaded.

---
 fdbclient/ServerKnobs.cpp                |  8 ++++++++
 fdbclient/ServerKnobs.h                  |  5 +++++
 fdbserver/IKeyValueStore.h               |  1 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 22 ++++++++++++++++++++++
 fdbserver/storageserver.actor.cpp        |  2 ++
 5 files changed, 38 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index de3d6d5443..f00caafbba 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -367,6 +367,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
 	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 					  0.0001 );
 	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
+	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
+	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
+	init( ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT,        50000000000 ); // 50GB, Commit waits.
+	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
+	// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
+	// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable
+	init( ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD,                    1 );
+	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,              5 );
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index cfe6c8d4d1..21fcac1495 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -298,6 +298,11 @@ public:
 	bool ROCKSDB_PERFCONTEXT_ENABLE; // Enable rocks perf context metrics. May cause performance overhead
 	double ROCKSDB_PERFCONTEXT_SAMPLE_RATE;
 	int ROCKSDB_MAX_SUBCOMPACTIONS;
+	int64_t ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
+	int64_t ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
+	int64_t ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT;
+	int ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD;
+	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
diff --git a/fdbserver/IKeyValueStore.h b/fdbserver/IKeyValueStore.h
index 4ae3aa7061..6217cada9c 100644
--- a/fdbserver/IKeyValueStore.h
+++ b/fdbserver/IKeyValueStore.h
@@ -46,6 +46,7 @@ public:
 	virtual KeyValueStoreType getType() const = 0;
 	virtual void set(KeyValueRef keyValue, const Arena* arena = nullptr) = 0;
 	virtual void clear(KeyRangeRef range, const Arena* arena = nullptr) = 0;
+	virtual Future<Void> canCommit() { return Void(); }
 	virtual Future<Void> commit(
 	    bool sequential = false) = 0; // returns when prior sets and clears are (atomically) durable
 
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 1e966b8b9b..a324501fb0 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -149,6 +149,12 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
 		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
 	}
+	if (SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT > 0) {
+		options.soft_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
+	}
+	if (SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) {
+		options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
+	}
 	// Compact sstables when there's too much deleted stuff.
 	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
 
@@ -1438,6 +1444,22 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 	}
 
+	// Checks and waits for few seconds if rocskdb is overloaded.
+	ACTOR Future<Void> checkRocksdbState(RocksDBKeyValueStore* self) {
+		state uint64_t estPendCompactBytes;
+		state int count = SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
+		self->db->GetIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
+		while (count && estPendCompactBytes > SERVER_KNOBS->ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT) {
+			wait(delay(SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD));
+			count--;
+			self->db->GetIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
+		}
+
+		return Void();
+	}
+
+	Future<Void> canCommit() override { return checkRocksdbState(this); }
+
 	Future<Void> commit(bool) override {
 		// If there is nothing to write, don't write.
 		if (writeBatch == nullptr) {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index a4fed0c584..769488c72a 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -204,6 +204,7 @@ struct StorageServerDisk {
 
 	Future<Void> getError() { return storage->getError(); }
 	Future<Void> init() { return storage->init(); }
+	Future<Void> canCommit() { return storage->canCommit(); }
 	Future<Void> commit() { return storage->commit(); }
 
 	// SOMEDAY: Put readNextKeyInclusive in IKeyValueStore
@@ -5588,6 +5589,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 
 		debug_advanceMaxCommittedVersion(data->thisServerID, newOldestVersion);
 		state double beforeStorageCommit = now();
+		wait(data->storage.canCommit());
 		state Future<Void> durable = data->storage.commit();
 		++data->counters.kvCommits;
 		state Future<Void> durableDelay = Void();

From 1555c0e2aa8ebb67987e311425b80c156a31cc08 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 4 Mar 2022 12:41:21 -0800
Subject: [PATCH 148/210] Fix conflict between two recent merges by using the
 new errorUnsuppressed function.

---
 fdbclient/NativeAPI.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 2a5c02ee8b..a21fe9331f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1058,7 +1058,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					    .detail("CachedReadVersion", cx->getCachedReadVersion())
 					    .detail("CachedTime", cx->getLastGrvTime());
 				} catch (Error& e) {
-					TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").error(e, true);
+					TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").errorUnsuppressed(e);
 					wait(tr.onError(e));
 				}
 			} else {
@@ -1069,7 +1069,7 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent(SevInfo, "BackgroundGrvUpdaterFailed").error(e, true);
+		TraceEvent(SevInfo, "BackgroundGrvUpdaterFailed").errorUnsuppressed(e);
 		throw;
 	}
 }

From 11fbd1564eb45a4ad250e3c439b2eedd4f0a891d Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 4 Mar 2022 12:44:52 -0800
Subject: [PATCH 149/210] Fix two more cases of error(e, true)

---
 fdbserver/workloads/SidebandSingle.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index e92adb75f2..8a7070d34c 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -161,7 +161,7 @@ struct SidebandSingleWorkload : TestWorkload {
 								wait(store(val2, tr2.get(messageKey)));
 								break;
 							} catch (Error& e) {
-								TraceEvent("DebugSidebandNoCacheError").error(e, true);
+								TraceEvent("DebugSidebandNoCacheError").errorUnsuppressed(e);
 								wait(tr2.onError(e));
 							}
 						}
@@ -178,7 +178,7 @@ struct SidebandSingleWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					TraceEvent("DebugSidebandCheckError").error(e, true);
+					TraceEvent("DebugSidebandCheckError").errorUnsuppressed(e);
 					wait(tr.onError(e));
 				}
 			}

From 2bd2ddd77942597b4f6e61b36bb797699c875b97 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Fri, 4 Mar 2022 12:49:40 -0800
Subject: [PATCH 150/210] Fix FastAlloc thread cleanup on OSX (#6485)

* Use ThreadData constructor and destructor

Instead of pthread api

* Make threadData a function static thread local variable

* Fix linux build
---
 bindings/c/CMakeLists.txt                     |   9 +-
 bindings/c/ThreadCleanup.cpp                  |  64 -----------
 .../c/test/unit/third_party/CMakeLists.txt    |   2 +-
 bindings/c/test/unit/unit_tests.cpp           |  13 +++
 flow/FastAlloc.cpp                            | 106 ++++++------------
 flow/FastAlloc.h                              |  27 +++--
 6 files changed, 69 insertions(+), 152 deletions(-)
 delete mode 100644 bindings/c/ThreadCleanup.cpp

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 78dce4cdc9..0c1bdbc554 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(FDB_C_SRCS
   fdb_c.cpp
-  foundationdb/fdb_c.h
-  ThreadCleanup.cpp)
+  foundationdb/fdb_c.h)
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
 
@@ -76,10 +75,8 @@ if(WIN32)
   set_property(SOURCE ${asm_file} PROPERTY LANGUAGE ASM_MASM)
 endif()
 
-# The tests don't build on windows and ARM macs
-# doctest doesn't seem to compile on ARM macs, we should
-# check later whether this works
-if(NOT WIN32 AND NOT IS_ARM_MAC)
+# The tests don't build on windows
+if(NOT WIN32)
   set(MAKO_SRCS
     test/mako/mako.c
     test/mako/mako.h
diff --git a/bindings/c/ThreadCleanup.cpp b/bindings/c/ThreadCleanup.cpp
deleted file mode 100644
index 86f61ba608..0000000000
--- a/bindings/c/ThreadCleanup.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * ThreadCleanup.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "flow/Platform.h"
-#include "flow/FastAlloc.h"
-
-#if defined(WIN32)
-
-#include <Windows.h>
-
-BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) {
-
-	if (reason == DLL_THREAD_DETACH)
-		releaseAllThreadMagazines();
-	return TRUE;
-}
-
-#elif defined(__unixish__)
-
-#ifdef __INTEL_COMPILER
-#pragma warning(disable : 2415)
-#endif
-
-static pthread_key_t threadDestructorKey;
-
-static void threadDestructor(void*) {
-	releaseAllThreadMagazines();
-}
-
-void registerThread() {
-	pthread_setspecific(threadDestructorKey, (const void*)1);
-}
-
-static int initThreadDestructorKey() {
-	if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) {
-		registerThread();
-		setFastAllocatorThreadInitFunction(&registerThread);
-	}
-
-	return 0;
-}
-
-static int threadDestructorKeyInit = initThreadDestructorKey();
-
-#else
-#error Port me!
-#endif
diff --git a/bindings/c/test/unit/third_party/CMakeLists.txt b/bindings/c/test/unit/third_party/CMakeLists.txt
index 6229abb0c9..014157553c 100644
--- a/bindings/c/test/unit/third_party/CMakeLists.txt
+++ b/bindings/c/test/unit/third_party/CMakeLists.txt
@@ -6,7 +6,7 @@ ExternalProject_Add(
     doctest
     PREFIX ${CMAKE_BINARY_DIR}/doctest
     GIT_REPOSITORY https://github.com/onqtam/doctest.git
-    GIT_TAG 1c8da00c978c19e00a434b2b1f854fcffc9fba35 # v2.4.0
+    GIT_TAG 8424be522357e68d8c6178375546bb0cf9d5f6b3 # v2.4.1
     TIMEOUT 10
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp
index 58a44de098..420a16dc3c 100644
--- a/bindings/c/test/unit/unit_tests.cpp
+++ b/bindings/c/test/unit/unit_tests.cpp
@@ -2358,6 +2358,19 @@ TEST_CASE("commit_does_not_reset") {
 	}
 }
 
+TEST_CASE("Fast alloc thread cleanup") {
+	// Try to cause an OOM if thread cleanup doesn't work
+	for (int i = 0; i < 50000; ++i) {
+		auto thread = std::thread([]() {
+			fdb::Transaction tr(db);
+			for (int s = 0; s < 11; ++s) {
+				tr.set(key("foo"), std::string(8 << s, '\x00'));
+			}
+		});
+		thread.join();
+	}
+}
+
 int main(int argc, char** argv) {
 	if (argc < 3) {
 		std::cout << "Unit tests for the FoundationDB C API.\n"
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index 6aaf295d69..3fe8d6af31 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -72,10 +72,13 @@
 #endif
 
 template <int Size>
-INIT_SEG thread_local typename FastAllocator<Size>::ThreadData FastAllocator<Size>::threadData;
+INIT_SEG thread_local typename FastAllocator<Size>::ThreadDataInit FastAllocator<Size>::threadDataInit;
 
 template <int Size>
-thread_local bool FastAllocator<Size>::threadInitialized = false;
+typename FastAllocator<Size>::ThreadData& FastAllocator<Size>::threadData() noexcept {
+	static thread_local ThreadData threadData;
+	return threadData;
+}
 
 #ifdef VALGRIND
 template <int Size>
@@ -108,14 +111,6 @@ bool valgrindPrecise() {
 template <int Size>
 void* FastAllocator<Size>::freelist = nullptr;
 
-typedef void (*ThreadInitFunction)();
-
-ThreadInitFunction threadInitFunction = 0; // See ThreadCleanup.cpp in the C binding
-void setFastAllocatorThreadInitFunction(ThreadInitFunction f) {
-	ASSERT(!threadInitFunction);
-	threadInitFunction = f;
-}
-
 std::atomic<int64_t> g_hugeArenaMemory(0);
 
 double hugeArenaLastLogged = 0;
@@ -310,9 +305,6 @@ static int64_t getSizeCode(int i) {
 
 template <int Size>
 void* FastAllocator<Size>::allocate() {
-	if (!threadInitialized) {
-		initThread();
-	}
 
 #if defined(USE_GPERFTOOLS) || defined(ADDRESS_SANITIZER)
 	// Some usages of FastAllocator require 4096 byte alignment.
@@ -327,7 +319,7 @@ void* FastAllocator<Size>::allocate() {
 #endif
 
 #if FASTALLOC_THREAD_SAFE
-	ThreadData& thr = threadData;
+	ThreadData& thr = threadData();
 	if (!thr.freelist) {
 		ASSERT(thr.count == 0);
 		if (thr.alternate) {
@@ -366,9 +358,6 @@ void* FastAllocator<Size>::allocate() {
 
 template <int Size>
 void FastAllocator<Size>::release(void* ptr) {
-	if (!threadInitialized) {
-		initThread();
-	}
 
 #if defined(USE_GPERFTOOLS) || defined(ADDRESS_SANITIZER)
 	return aligned_free(ptr);
@@ -381,7 +370,7 @@ void FastAllocator<Size>::release(void* ptr) {
 #endif
 
 #if FASTALLOC_THREAD_SAFE
-	ThreadData& thr = threadData;
+	ThreadData& thr = threadData();
 	if (thr.count == magazine_size) {
 		if (thr.alternate) // Two full magazines, return one
 			releaseMagazine(thr.alternate);
@@ -462,39 +451,33 @@ void FastAllocator<Size>::check(void* ptr, bool alloc) {
 }
 
 template <int Size>
-void FastAllocator<Size>::initThread() {
-	threadInitialized = true;
-	if (threadInitFunction) {
-		threadInitFunction();
-	}
-
+FastAllocator<Size>::ThreadData::ThreadData() {
 	globalData()->activeThreads.fetch_add(1);
-
-	threadData.freelist = nullptr;
-	threadData.alternate = nullptr;
-	threadData.count = 0;
+	freelist = nullptr;
+	alternate = nullptr;
+	count = 0;
 }
 
 template <int Size>
 void FastAllocator<Size>::getMagazine() {
-	ASSERT(threadInitialized);
-	ASSERT(!threadData.freelist && !threadData.alternate && threadData.count == 0);
+	ThreadData& thr = threadData();
+	ASSERT(!thr.freelist && !thr.alternate && thr.count == 0);
 
 	EnterCriticalSection(&globalData()->mutex);
 	if (globalData()->magazines.size()) {
 		void* m = globalData()->magazines.back();
 		globalData()->magazines.pop_back();
 		LeaveCriticalSection(&globalData()->mutex);
-		threadData.freelist = m;
-		threadData.count = magazine_size;
+		thr.freelist = m;
+		thr.count = magazine_size;
 		return;
 	} else if (globalData()->partial_magazines.size()) {
 		std::pair<int, void*> p = globalData()->partial_magazines.back();
 		globalData()->partial_magazines.pop_back();
 		globalData()->partialMagazineUnallocatedMemory -= p.first * Size;
 		LeaveCriticalSection(&globalData()->mutex);
-		threadData.freelist = p.second;
-		threadData.count = p.first;
+		thr.freelist = p.second;
+		thr.count = p.first;
 		return;
 	}
 	globalData()->totalMemory.fetch_add(magazine_size * Size);
@@ -546,55 +529,32 @@ void FastAllocator<Size>::getMagazine() {
 
 	block[(magazine_size - 1) * PSize + 1] = block[(magazine_size - 1) * PSize] = nullptr;
 	check(&block[(magazine_size - 1) * PSize], false);
-	threadData.freelist = block;
-	threadData.count = magazine_size;
+	thr.freelist = block;
+	thr.count = magazine_size;
 }
 template <int Size>
 void FastAllocator<Size>::releaseMagazine(void* mag) {
-	ASSERT(threadInitialized);
 	EnterCriticalSection(&globalData()->mutex);
 	globalData()->magazines.push_back(mag);
 	LeaveCriticalSection(&globalData()->mutex);
 }
 template <int Size>
-void FastAllocator<Size>::releaseThreadMagazines() {
-	if (threadInitialized) {
-		threadInitialized = false;
-		ThreadData& thr = threadData;
-
-		EnterCriticalSection(&globalData()->mutex);
-		if (thr.freelist || thr.alternate) {
-			if (thr.freelist) {
-				ASSERT(thr.count > 0 && thr.count <= magazine_size);
-				globalData()->partial_magazines.emplace_back(thr.count, thr.freelist);
-				globalData()->partialMagazineUnallocatedMemory += thr.count * Size;
-			}
-			if (thr.alternate) {
-				globalData()->magazines.push_back(thr.alternate);
-			}
-		}
-		globalData()->activeThreads.fetch_add(-1);
-		LeaveCriticalSection(&globalData()->mutex);
-
-		thr.count = 0;
-		thr.alternate = nullptr;
-		thr.freelist = nullptr;
+FastAllocator<Size>::ThreadData::~ThreadData() {
+	EnterCriticalSection(&globalData()->mutex);
+	if (freelist) {
+		ASSERT_ABORT(count > 0 && count <= magazine_size);
+		globalData()->partial_magazines.emplace_back(count, freelist);
+		globalData()->partialMagazineUnallocatedMemory += count * Size;
 	}
-}
+	if (alternate) {
+		globalData()->magazines.push_back(alternate);
+	}
+	globalData()->activeThreads.fetch_add(-1);
+	LeaveCriticalSection(&globalData()->mutex);
 
-void releaseAllThreadMagazines() {
-	FastAllocator<16>::releaseThreadMagazines();
-	FastAllocator<32>::releaseThreadMagazines();
-	FastAllocator<64>::releaseThreadMagazines();
-	FastAllocator<96>::releaseThreadMagazines();
-	FastAllocator<128>::releaseThreadMagazines();
-	FastAllocator<256>::releaseThreadMagazines();
-	FastAllocator<512>::releaseThreadMagazines();
-	FastAllocator<1024>::releaseThreadMagazines();
-	FastAllocator<2048>::releaseThreadMagazines();
-	FastAllocator<4096>::releaseThreadMagazines();
-	FastAllocator<8192>::releaseThreadMagazines();
-	FastAllocator<16384>::releaseThreadMagazines();
+	count = 0;
+	alternate = nullptr;
+	freelist = nullptr;
 }
 
 int64_t getTotalUnusedAllocatedMemory() {
diff --git a/flow/FastAlloc.h b/flow/FastAlloc.h
index 55e5731fd5..6fbc6ce0af 100644
--- a/flow/FastAlloc.h
+++ b/flow/FastAlloc.h
@@ -114,8 +114,6 @@ public:
 	static long long getApproximateMemoryUnused();
 	static long long getActiveThreads();
 
-	static void releaseThreadMagazines();
-
 #ifdef ALLOC_INSTRUMENTATION
 	static volatile int32_t pageCount;
 #endif
@@ -134,9 +132,26 @@ private:
 		void* freelist;
 		int count; // there are count items on freelist
 		void* alternate; // alternate is either a full magazine, or an empty one
+		ThreadData();
+		~ThreadData();
 	};
-	static thread_local ThreadData threadData;
-	static thread_local bool threadInitialized;
+	struct ThreadDataInit {
+		ThreadDataInit() { threadData(); }
+	};
+	// Used to try to initialize threadData as early as possible. It's still
+	// possible that a static thread local variable (that owns fast-allocated
+	// memory) could be constructed before threadData, in which case threadData
+	// would be destroyed by the time that variable's destructor attempts to free.
+	// This is undefined behavior if this happens, which is why we want to
+	// initialize threadData as early as possible.
+	static thread_local ThreadDataInit threadDataInit;
+	// Used to access threadData. Returning a reference to a function-level
+	// static guarantees that threadData will be constructed before it's
+	// accessed here. Furthermore, if accessing threadData from a static thread
+	// local variable's constructor, this guarantees that threadData will
+	// outlive this object, since destruction order is the reverse of
+	// construction order.
+	static ThreadData& threadData() noexcept;
 	static GlobalData* globalData() noexcept {
 #ifdef VALGRIND
 		ANNOTATE_RWLOCK_ACQUIRED(vLock, 1);
@@ -151,7 +166,6 @@ private:
 	}
 	static void* freelist;
 
-	static void initThread();
 	static void getMagazine();
 	static void releaseMagazine(void*);
 };
@@ -160,9 +174,6 @@ extern std::atomic<int64_t> g_hugeArenaMemory;
 void hugeArenaSample(int size);
 void releaseAllThreadMagazines();
 int64_t getTotalUnusedAllocatedMemory();
-void setFastAllocatorThreadInitFunction(
-    void (*)()); // The given function will be called at least once in each thread that allocates from a FastAllocator.
-                 // Currently just one such function is tracked.
 
 inline constexpr int nextFastAllocatedSize(int x) {
 	assert(x > 0 && x <= 8192);

From 41d1e4b5d53b60f397207ec5d0d1845e1792c308 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 1 Mar 2022 08:57:01 -0800
Subject: [PATCH 151/210] Fix several instances where we were converting keys,
 ranges, and key selectors into strings before passing them into trace events.
 This had the effect of double escaping certain characters.

---
 fdbclient/FDBTypes.h                          |  5 ++
 fdbclient/SpecialKeySpace.actor.cpp           |  4 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 10 ++-
 .../DataDistributionMetrics.actor.cpp         | 24 +++----
 .../workloads/FuzzApiCorrectness.actor.cpp    |  6 +-
 .../SpecialKeySpaceCorrectness.actor.cpp      | 20 +++---
 fdbserver/workloads/WriteDuringRead.actor.cpp | 72 +++++++++----------
 7 files changed, 72 insertions(+), 69 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 8a9bfe6ac0..4c4d1b46e5 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -593,6 +593,11 @@ inline bool selectorInRange(KeySelectorRef const& sel, KeyRangeRef const& range)
 	return sel.getKey() >= range.begin && (sel.isBackward() ? sel.getKey() <= range.end : sel.getKey() < range.end);
 }
 
+template <>
+struct Traceable<KeySelectorRef> : std::true_type {
+	static std::string toString(const KeySelectorRef& value) { return value.toString(); }
+};
+
 template <class Val>
 struct KeyRangeWith : KeyRange {
 	Val value;
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index af4fe3e470..fd1087c576 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -340,8 +340,8 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 				moduleBoundary = beginIter->range();
 		} else {
 			TraceEvent(SevInfo, "SpecialKeyCrossModuleRead")
-			    .detail("Begin", begin.toString())
-			    .detail("End", end.toString())
+			    .detail("Begin", begin)
+			    .detail("End", end)
 			    .detail("BoundaryBegin", beginIter->begin())
 			    .detail("BoundaryEnd", beginIter->end());
 			throw special_keys_cross_module_read();
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index ef4f1a4599..d7b526b6f6 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -430,9 +430,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (serverIndices.size()) {
 				KeyRangeRef range(cacheKey[k].key, (k < cacheKey.size() - 1) ? cacheKey[k + 1].key : allKeys.end);
 				cachedKeysLocationMap.insert(range, cacheServerInterfaces);
-				TraceEvent(SevDebug, "CheckCacheConsistency")
-				    .detail("CachedRange", range.toString())
-				    .detail("Index", k);
+				TraceEvent(SevDebug, "CheckCacheConsistency").detail("CachedRange", range).detail("Index", k);
 			}
 		}
 		// Second, insert corresponding storage servers into the list
@@ -545,8 +543,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 					wait(waitForAll(keyValueFutures));
 					TraceEvent(SevDebug, "CheckCacheConsistencyComparison")
-					    .detail("Begin", req.begin.toString())
-					    .detail("End", req.end.toString())
+					    .detail("Begin", req.begin)
+					    .detail("End", req.end)
 					    .detail("SSInterfaces", describe(iter_ss));
 
 					// Read the resulting entries
@@ -712,7 +710,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						begin = firstGreaterThan(result[result.size() - 1].key);
 						ASSERT(begin.getKey() != allKeys.end);
 						lastStartSampleKey = lastSampleKey;
-						TraceEvent(SevDebug, "CacheConsistencyCheckNextBeginKey").detail("Key", begin.toString());
+						TraceEvent(SevDebug, "CacheConsistencyCheckNextBeginKey").detail("Key", begin);
 					} else
 						break;
 				} catch (Error& e) {
diff --git a/fdbserver/workloads/DataDistributionMetrics.actor.cpp b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
index 39dcc1678a..73c2af5ee6 100644
--- a/fdbserver/workloads/DataDistributionMetrics.actor.cpp
+++ b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
@@ -96,30 +96,30 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "Result mismatches the given begin selector")
 						    .detail("Size", result.size())
-						    .detail("FirstKey", result[0].key.toString())
-						    .detail("SecondKey", result[1].key.toString())
-						    .detail("BeginKeySelector", begin.toString());
+						    .detail("FirstKey", result[0].key)
+						    .detail("SecondKey", result[1].key)
+						    .detail("BeginKeySelector", begin);
 					}
 					if (result[result.size() - 1].key < end.getKey() || result[result.size() - 2].key >= end.getKey()) {
 						++self->errors;
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "Result mismatches the given end selector")
 						    .detail("Size", result.size())
-						    .detail("FirstKey", result[result.size() - 1].key.toString())
-						    .detail("SecondKey", result[result.size() - 2].key.toString())
-						    .detail("EndKeySelector", end.toString());
+						    .detail("FirstKey", result[result.size() - 1].key)
+						    .detail("SecondKey", result[result.size() - 2].key)
+						    .detail("EndKeySelector", end);
 					}
 					// Debugging traces
 					// TraceEvent(SevDebug, "DDMetricsConsistencyTest")
 					// 	    .detail("Size", result.size())
-					// 	    .detail("FirstKey", result[0].key.toString())
-					// 	    .detail("SecondKey", result[1].key.toString())
-					// 	    .detail("BeginKeySelector", begin.toString());
+					// 	    .detail("FirstKey", result[0].key)
+					// 	    .detail("SecondKey", result[1].key)
+					// 	    .detail("BeginKeySelector", begin);
 					// TraceEvent(SevDebug, "DDMetricsConsistencyTest")
 					// 	    .detail("Size", result.size())
-					// 	    .detail("LastKey", result[result.size() - 1].key.toString())
-					// 	    .detail("SecondLastKey", result[result.size() - 2].key.toString())
-					// 	    .detail("EndKeySelector", end.toString());
+					// 	    .detail("LastKey", result[result.size() - 1].key)
+					// 	    .detail("SecondLastKey", result[result.size() - 2].key)
+					// 	    .detail("EndKeySelector", end);
 				}
 			} catch (Error& e) {
 				// Ignore timed_out error and cross_module_read, the end key selector may read through the end
diff --git a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
index 5efdfe62e0..8866eacb20 100644
--- a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
@@ -675,7 +675,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 		void augmentTrace(TraceEvent& e) const override {
 			base_type::augmentTrace(e);
-			e.detail("KeySel", keysel.toString());
+			e.detail("KeySel", keysel);
 		}
 	};
 
@@ -726,7 +726,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 		void augmentTrace(TraceEvent& e) const override {
 			base_type::augmentTrace(e);
-			e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString()).detail("Limit", limit);
+			e.detail("KeySel1", keysel1).detail("KeySel2", keysel2).detail("Limit", limit);
 		}
 	};
 
@@ -769,7 +769,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 		void augmentTrace(TraceEvent& e) const override {
 			base_type::augmentTrace(e);
-			e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString());
+			e.detail("KeySel1", keysel1).detail("KeySel2", keysel2);
 			std::stringstream ss;
 			ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")";
 			e.detail("Limits", ss.str());
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index d75ca27f14..8f7b2db8dc 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -144,8 +144,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			if (!self->compareRangeResult(correctResult, testResult)) {
 				TraceEvent(SevError, "TestFailure")
 				    .detail("Reason", "Results from getRange are inconsistent")
-				    .detail("Begin", begin.toString())
-				    .detail("End", end.toString())
+				    .detail("Begin", begin)
+				    .detail("End", end)
 				    .detail("LimitRows", limit.rows)
 				    .detail("LimitBytes", limit.bytes)
 				    .detail("Reverse", reverse);
@@ -185,8 +185,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			if (!self->compareRangeResult(correctRywResult, testRywResult)) {
 				TraceEvent(SevError, "TestFailure")
 				    .detail("Reason", "Results from getRange(ryw) are inconsistent")
-				    .detail("Begin", begin.toString())
-				    .detail("End", end.toString())
+				    .detail("Begin", begin)
+				    .detail("End", end)
 				    .detail("LimitRows", limit.rows)
 				    .detail("LimitBytes", limit.bytes)
 				    .detail("Reverse", reverse);
@@ -569,8 +569,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					    .detail("TestKey", test_iter->key)
 					    .detail("CorrectValue", correct_iter->value)
 					    .detail("TestValue", test_iter->value)
-					    .detail("Begin", begin.toString())
-					    .detail("End", end.toString())
+					    .detail("Begin", begin)
+					    .detail("End", end)
 					    .detail("Ryw", ryw);
 					had_error = true;
 					++self->wrongResults;
@@ -584,8 +584,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    .detail("ConflictType", read ? "read" : "write")
 				    .detail("CorrectKey", correct_iter->key)
 				    .detail("CorrectValue", correct_iter->value)
-				    .detail("Begin", begin.toString())
-				    .detail("End", end.toString())
+				    .detail("Begin", begin)
+				    .detail("End", end)
 				    .detail("Ryw", ryw);
 				++correct_iter;
 				had_error = true;
@@ -597,8 +597,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    .detail("ConflictType", read ? "read" : "write")
 				    .detail("TestKey", test_iter->key)
 				    .detail("TestValue", test_iter->value)
-				    .detail("Begin", begin.toString())
-				    .detail("End", end.toString())
+				    .detail("Begin", begin)
+				    .detail("End", end)
 				    .detail("Ryw", ryw);
 				++test_iter;
 				had_error = true;
diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index c41bf68f4a..8b77a391c6 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -170,12 +170,12 @@ struct WriteDuringReadWorkload : TestWorkload {
 				res = allKeys.end;
 			if (res != memRes) {
 				TraceEvent(SevError, "WDRGetKeyWrongResult", randomID)
-				    .detail("Key", printable(key.getKey()))
+				    .detail("Key", key.getKey())
 				    .detail("Offset", key.offset)
 				    .detail("OrEqual", key.orEqual)
 				    .detail("Snapshot", snapshot)
-				    .detail("MemoryResult", printable(memRes))
-				    .detail("DbResult", printable(res));
+				    .detail("MemoryResult", memRes)
+				    .detail("DbResult", res);
 				self->success = false;
 			}
 			return Void();
@@ -197,7 +197,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 	                                                  Reverse reverse) {
 		Key beginKey = memoryGetKey(db, begin);
 		Key endKey = memoryGetKey(db, end);
-		//TraceEvent("WDRGetRange").detail("Begin", printable(beginKey)).detail("End", printable(endKey));
+		//TraceEvent("WDRGetRange").detail("Begin", beginKey).detail("End", endKey);
 		if (beginKey >= endKey)
 			return Standalone<VectorRef<KeyValueRef>>();
 
@@ -232,8 +232,8 @@ struct WriteDuringReadWorkload : TestWorkload {
 	                                      bool* doingCommit,
 	                                      int64_t* memLimit) {
 		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		/*TraceEvent("WDRGetRange", randomID).detail("BeginKey", printable(begin.getKey())).detail("BeginOffset", begin.offset).detail("BeginOrEqual", begin.orEqual)
-		    .detail("EndKey", printable(end.getKey())).detail("EndOffset", end.offset).detail("EndOrEqual", end.orEqual)
+		/*TraceEvent("WDRGetRange", randomID).detail("BeginKey", begin.getKey()).detail("BeginOffset", begin.offset).detail("BeginOrEqual", begin.orEqual)
+		    .detail("EndKey", end.getKey()).detail("EndOffset", end.offset).detail("EndOrEqual", end.orEqual)
 		    .detail("Limit", limit.rows).detail("Snapshot", snapshot).detail("Reverse",
 		   reverse).detail("ReadYourWritesDisabled", readYourWritesDisabled);*/
 
@@ -277,10 +277,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 			if (!limit.hasByteLimit() && systemKeyCount == 0) {
 				if (res.size() != memRes.size()) {
 					TraceEvent(SevError, "WDRGetRangeWrongResult", randomID)
-					    .detail("BeginKey", printable(begin.getKey()))
+					    .detail("BeginKey", begin.getKey())
 					    .detail("BeginOffset", begin.offset)
 					    .detail("BeginOrEqual", begin.orEqual)
-					    .detail("EndKey", printable(end.getKey()))
+					    .detail("EndKey", end.getKey())
 					    .detail("EndOffset", end.offset)
 					    .detail("EndOrEqual", end.orEqual)
 					    .detail("LimitRows", limit.rows)
@@ -298,10 +298,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 				for (int i = 0; i < res.size(); i++) {
 					if (res[i] != memRes[i]) {
 						TraceEvent(SevError, "WDRGetRangeWrongResult", randomID)
-						    .detail("BeginKey", printable(begin.getKey()))
+						    .detail("BeginKey", begin.getKey())
 						    .detail("BeginOffset", begin.offset)
 						    .detail("BeginOrEqual", begin.orEqual)
-						    .detail("EndKey", printable(end.getKey()))
+						    .detail("EndKey", end.getKey())
 						    .detail("EndOffset", end.offset)
 						    .detail("EndOrEqual", end.orEqual)
 						    .detail("LimitRows", limit.rows)
@@ -310,8 +310,8 @@ struct WriteDuringReadWorkload : TestWorkload {
 						    .detail("Reverse", reverse)
 						    .detail("Size", memRes.size())
 						    .detail("WrongLocation", i)
-						    .detail("MemoryResultKey", printable(memRes[i].key))
-						    .detail("DbResultKey", printable(res[i].key))
+						    .detail("MemoryResultKey", memRes[i].key)
+						    .detail("DbResultKey", res[i].key)
 						    .detail("MemoryResultValueSize", memRes[i].value.size())
 						    .detail("DbResultValueSize", res[i].value.size())
 						    .detail("ReadYourWritesDisabled", readYourWritesDisabled);
@@ -323,10 +323,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 				if (res.size() > memRes.size() || (res.size() < memRes.size() && !res.more) ||
 				    (res.size() == 0 && res.more && !resized)) {
 					TraceEvent(SevError, "WDRGetRangeWrongResult", randomID)
-					    .detail("BeginKey", printable(begin.getKey()))
+					    .detail("BeginKey", begin.getKey())
 					    .detail("BeginOffset", begin.offset)
 					    .detail("BeginOrEqual", begin.orEqual)
-					    .detail("EndKey", printable(end.getKey()))
+					    .detail("EndKey", end.getKey())
 					    .detail("EndOffset", end.offset)
 					    .detail("EndOrEqual", end.orEqual)
 					    .detail("LimitRows", limit.rows)
@@ -346,10 +346,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 				for (int i = 0; i < res.size(); i++) {
 					if (res[i] != memRes[i]) {
 						TraceEvent(SevError, "WDRGetRangeWrongResult", randomID)
-						    .detail("BeginKey", printable(begin.getKey()))
+						    .detail("BeginKey", begin.getKey())
 						    .detail("BeginOffset", begin.offset)
 						    .detail("BeginOrEqual", begin.orEqual)
-						    .detail("EndKey", printable(end.getKey()))
+						    .detail("EndKey", end.getKey())
 						    .detail("EndOffset", end.offset)
 						    .detail("EndOrEqual", end.orEqual)
 						    .detail("LimitRows", limit.rows)
@@ -358,8 +358,8 @@ struct WriteDuringReadWorkload : TestWorkload {
 						    .detail("Reverse", reverse)
 						    .detail("Size", memRes.size())
 						    .detail("WrongLocation", i)
-						    .detail("MemoryResultKey", printable(memRes[i].key))
-						    .detail("DbResultKey", printable(res[i].key))
+						    .detail("MemoryResultKey", memRes[i].key)
+						    .detail("DbResultKey", res[i].key)
 						    .detail("MemoryResultValueSize", memRes[i].value.size())
 						    .detail("DbResultValueSize", res[i].value.size())
 						    .detail("ReadYourWritesDisabled", readYourWritesDisabled)
@@ -409,7 +409,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 			*memLimit += memRes.expectedSize();
 			if (res != memRes) {
 				TraceEvent(SevError, "WDRGetWrongResult", randomID)
-				    .detail("Key", printable(key))
+				    .detail("Key", key)
 				    .detail("Snapshot", snapshot)
 				    .detail("MemoryResult", memRes.present() ? memRes.get().size() : -1)
 				    .detail("DbResult", res.present() ? res.get().size() : -1)
@@ -439,7 +439,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 		if (readYourWritesDisabled) // Only tests RYW activated watches
 			return Void();
 
-		//TraceEvent("WDRWatch", randomID).detail("Key", printable(key));
+		//TraceEvent("WDRWatch", randomID).detail("Key", key);
 		try {
 			state int changeNum = self->changeCount[key];
 			state Optional<Value> memRes = self->memoryGet(&self->memoryDatabase, key);
@@ -452,7 +452,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 						if (changeNum == self->changeCount[key]) {
 							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
 							    .detail("Reason", "Triggered without changing")
-							    .detail("Key", printable(key))
+							    .detail("Key", key)
 							    .detail("Value", changeNum)
 							    .detail("DuringCommit", *doingCommit);
 						}
@@ -462,9 +462,9 @@ struct WriteDuringReadWorkload : TestWorkload {
 						if (memRes != memRes2) {
 							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
 							    .detail("Reason", "Changed without triggering")
-							    .detail("Key", printable(key))
-							    .detail("Value1", printable(memRes))
-							    .detail("Value2", printable(memRes2));
+							    .detail("Key", key)
+							    .detail("Value1", memRes)
+							    .detail("Value2", memRes2);
 						}
 					}
 				}
@@ -511,8 +511,8 @@ struct WriteDuringReadWorkload : TestWorkload {
 					if (transactionIter->begin() != addedIter->begin() ||
 					    transactionIter->value() != addedIter->value()) {
 						TraceEvent(SevError, "WriteConflictError")
-						    .detail("TransactionKey", printable(transactionIter->begin()))
-						    .detail("AddedKey", printable(addedIter->begin()))
+						    .detail("TransactionKey", transactionIter->begin())
+						    .detail("AddedKey", addedIter->begin())
 						    .detail("TransactionVal", transactionIter->value())
 						    .detail("AddedVal", addedIter->value());
 						failed = true;
@@ -530,12 +530,12 @@ struct WriteDuringReadWorkload : TestWorkload {
 					for (transactionIter = transactionRanges.begin(); transactionIter != transactionRanges.end();
 					     ++transactionIter) {
 						TraceEvent("WCRTransaction")
-						    .detail("Range", printable(transactionIter.range()))
+						    .detail("Range", transactionIter.range())
 						    .detail("Value", transactionIter.value());
 					}
 					for (addedIter = addedRanges.begin(); addedIter != addedRanges.end(); ++addedIter) {
 						TraceEvent("WCRAdded")
-						    .detail("Range", printable(addedIter.range()))
+						    .detail("Range", addedIter.range())
 						    .detail("Value", addedIter.value());
 					}
 				}
@@ -854,7 +854,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 								KeyRange range = self->getRandomRange(self->maxClearSize);
 								self->changeCount.insert(range, changeNum++);
 								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRClearRange").detail("Begin", printable(range)).detail("NoConflict", noConflict);
+								//TraceEvent("WDRClearRange").detail("Begin", range).detail("NoConflict", noConflict);
 								if (noConflict)
 									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 								tr.clear(range);
@@ -880,7 +880,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 								Key key = self->getRandomKey();
 								self->changeCount.insert(key, changeNum++);
 								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRClear").detail("Key", printable(key)).detail("NoConflict", noConflict);
+								//TraceEvent("WDRClear").detail("Key", key).detail("NoConflict", noConflict);
 								if (noConflict)
 									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 								tr.clear(key);
@@ -895,7 +895,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 								    &tr, self->getRandomKey(), readYourWritesDisabled, self, &doingCommit, &memLimit));
 							} else if (operationType == 7 && !disableWriteConflictRange) {
 								KeyRange range = self->getRandomRange(self->nodes);
-								//TraceEvent("WDRAddWriteConflict").detail("Range", printable(range));
+								//TraceEvent("WDRAddWriteConflict").detail("Range", range);
 								tr.addWriteConflictRange(range);
 								KeyRangeRef conflict(
 								    range.begin.substr(0,
@@ -949,7 +949,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 									                                            tr.getCachedReadVersion().orDefault(0),
 									                                            normalKeys.end);
 									self->changeCount.insert(range, changeNum++);
-									//TraceEvent("WDRVersionStamp").detail("VersionStampKey", printable(versionStampKey)).detail("Range", printable(range));
+									//TraceEvent("WDRVersionStamp").detail("VersionStampKey", versionStampKey).detail("Range", range);
 									tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
 									tr.clear(range);
 									KeyRangeRef conflict(
@@ -1000,11 +1000,11 @@ struct WriteDuringReadWorkload : TestWorkload {
 									}
 									self->changeCount.insert(key, changeNum++);
 									bool noConflict = deterministicRandom()->random01() < 0.5;
-									//TraceEvent("WDRAtomicOp").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
+									//TraceEvent("WDRAtomicOp").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
 									if (noConflict)
 										tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 									tr.atomicOp(key, value, opType);
-									//TraceEvent("WDRAtomicOpSuccess").detail("Key", printable(key)).detail("Value", value.size());
+									//TraceEvent("WDRAtomicOpSuccess").detail("Key", key).detail("Value", value.size());
 									if (!noConflict && key.size() <= (key.startsWith(systemKeys.begin)
 									                                      ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
 									                                      : CLIENT_KNOBS->KEY_SIZE_LIMIT))
@@ -1021,7 +1021,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 								Value value = self->getRandomValue();
 								self->changeCount.insert(key, changeNum++);
 								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRSet").detail("Key", printable(key)).detail("Value", value.size()).detail("NoConflict", noConflict);
+								//TraceEvent("WDRSet").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
 								if (noConflict)
 									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 								tr.set(key, value);
@@ -1029,7 +1029,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 								                                      ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
 								                                      : CLIENT_KNOBS->KEY_SIZE_LIMIT))
 									self->addedConflicts.insert(key, true);
-								//TraceEvent("WDRSetSuccess").detail("Key", printable(key)).detail("Value", value.size());
+								//TraceEvent("WDRSetSuccess").detail("Key", key).detail("Value", value.size());
 								self->memoryDatabase[key] = value;
 							}
 						} catch (Error& e) {

From d45bae11b7049fa79c699a8dd3e25158aed50636 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 4 Mar 2022 12:23:16 -0800
Subject: [PATCH 152/210] Fix substring call when reporting details about a
 trace event that was longer than the legal limit.

---
 flow/Trace.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 0afbc60c85..1b3eefc5fd 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -1022,7 +1022,7 @@ BaseTraceEvent& BaseTraceEvent::detailImpl(std::string&& key, std::string&& valu
 		if (maxEventLength >= 0 && fields.sizeBytes() > maxEventLength) {
 			TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways, "TraceEventOverflow")
 			    .setMaxEventLength(1000)
-			    .detail("TraceFirstBytes", fields.toString().substr(300));
+			    .detail("TraceFirstBytes", fields.toString().substr(0, 300));
 			enabled = false;
 		}
 		--g_allocation_tracing_disabled;

From ad792630aae65725e238bd2be8d97e86d46b57c5 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 4 Mar 2022 14:06:16 -0800
Subject: [PATCH 153/210] Fix formatting

---
 fdbserver/workloads/WriteDuringRead.actor.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index 8b77a391c6..151b66a9c6 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -534,9 +534,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 						    .detail("Value", transactionIter.value());
 					}
 					for (addedIter = addedRanges.begin(); addedIter != addedRanges.end(); ++addedIter) {
-						TraceEvent("WCRAdded")
-						    .detail("Range", addedIter.range())
-						    .detail("Value", addedIter.value());
+						TraceEvent("WCRAdded").detail("Range", addedIter.range()).detail("Value", addedIter.value());
 					}
 				}
 			}

From 212e5147b9611c32b0fb676b68beb41336a8e465 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 4 Mar 2022 10:41:26 -0800
Subject: [PATCH 154/210] Update fmt from 8.0.1 to 8.1.1

8.1.1 provides fmt::fprintf().
---
 contrib/CMakeLists.txt                        |    2 +-
 contrib/fmt-8.0.1/.clang-format               |    1 -
 contrib/fmt-8.0.1/include/fmt/ostream.h       |  181 ---
 contrib/fmt-8.0.1/include/fmt/ranges.h        |  468 -------
 contrib/fmt-8.1.1/.clang-format               |    8 +
 .../{fmt-8.0.1 => fmt-8.1.1}/CMakeLists.txt   |   17 +-
 contrib/{fmt-8.0.1 => fmt-8.1.1}/LICENSE.rst  |    0
 .../include/fmt/args.h                        |    2 +
 .../include/fmt/chrono.h                      | 1181 ++++++++++++++---
 .../include/fmt/color.h                       |   35 +-
 .../include/fmt/compile.h                     |   25 +-
 .../include/fmt/core.h                        |  746 +++++++----
 .../include/fmt/format-inl.h                  |  593 +++++----
 .../include/fmt/format.h                      |  906 ++++++++-----
 .../include/fmt/locale.h                      |    0
 .../{fmt-8.0.1 => fmt-8.1.1}/include/fmt/os.h |   48 +-
 contrib/fmt-8.1.1/include/fmt/ostream.h       |  135 ++
 .../include/fmt/printf.h                      |   19 +-
 contrib/fmt-8.1.1/include/fmt/ranges.h        |  793 +++++++++++
 .../include/fmt/xchar.h                       |   10 +-
 contrib/{fmt-8.0.1 => fmt-8.1.1}/src/fmt.cc   |    1 -
 .../{fmt-8.0.1 => fmt-8.1.1}/src/format.cc    |   46 +
 contrib/{fmt-8.0.1 => fmt-8.1.1}/src/os.cc    |   17 +-
 .../support/cmake/FindSetEnv.cmake            |    0
 .../support/cmake/JoinPaths.cmake             |    0
 .../support/cmake/cxx14.cmake                 |    0
 .../support/cmake/fmt-config.cmake.in         |    0
 .../support/cmake/fmt.pc.in                   |    0
 28 files changed, 3447 insertions(+), 1787 deletions(-)
 delete mode 100644 contrib/fmt-8.0.1/.clang-format
 delete mode 100644 contrib/fmt-8.0.1/include/fmt/ostream.h
 delete mode 100644 contrib/fmt-8.0.1/include/fmt/ranges.h
 create mode 100644 contrib/fmt-8.1.1/.clang-format
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/CMakeLists.txt (96%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/LICENSE.rst (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/args.h (99%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/chrono.h (51%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/color.h (95%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/compile.h (96%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/core.h (82%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/format-inl.h (90%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/format.h (79%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/locale.h (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/os.h (93%)
 create mode 100644 contrib/fmt-8.1.1/include/fmt/ostream.h
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/printf.h (97%)
 create mode 100644 contrib/fmt-8.1.1/include/fmt/ranges.h
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/include/fmt/xchar.h (97%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/src/fmt.cc (99%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/src/format.cc (52%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/src/os.cc (98%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/support/cmake/FindSetEnv.cmake (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/support/cmake/JoinPaths.cmake (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/support/cmake/cxx14.cmake (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/support/cmake/fmt-config.cmake.in (100%)
 rename contrib/{fmt-8.0.1 => fmt-8.1.1}/support/cmake/fmt.pc.in (100%)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 47705943be..f2da5835fc 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_subdirectory(fmt-8.0.1)
+add_subdirectory(fmt-8.1.1)
 if(NOT WIN32)
   add_subdirectory(debug_determinism)
   add_subdirectory(monitoring)
diff --git a/contrib/fmt-8.0.1/.clang-format b/contrib/fmt-8.0.1/.clang-format
deleted file mode 100644
index e3845288a2..0000000000
--- a/contrib/fmt-8.0.1/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/contrib/fmt-8.0.1/include/fmt/ostream.h b/contrib/fmt-8.0.1/include/fmt/ostream.h
deleted file mode 100644
index d66248a601..0000000000
--- a/contrib/fmt-8.0.1/include/fmt/ostream.h
+++ /dev/null
@@ -1,181 +0,0 @@
-// Formatting library for C++ - std::ostream support
-//
-// Copyright (c) 2012 - present, Victor Zverovich
-// All rights reserved.
-//
-// For the license information refer to format.h.
-
-#ifndef FMT_OSTREAM_H_
-#define FMT_OSTREAM_H_
-
-#include <ostream>
-
-#include "format.h"
-
-FMT_BEGIN_NAMESPACE
-
-template <typename Char> class basic_printf_parse_context;
-template <typename OutputIt, typename Char> class basic_printf_context;
-
-namespace detail {
-
-template <class Char> class formatbuf : public std::basic_streambuf<Char> {
- private:
-  using int_type = typename std::basic_streambuf<Char>::int_type;
-  using traits_type = typename std::basic_streambuf<Char>::traits_type;
-
-  buffer<Char>& buffer_;
-
- public:
-  formatbuf(buffer<Char>& buf) : buffer_(buf) {}
-
- protected:
-  // The put-area is actually always empty. This makes the implementation
-  // simpler and has the advantage that the streambuf and the buffer are always
-  // in sync and sputc never writes into uninitialized memory. The obvious
-  // disadvantage is that each call to sputc always results in a (virtual) call
-  // to overflow. There is no disadvantage here for sputn since this always
-  // results in a call to xsputn.
-
-  int_type overflow(int_type ch = traits_type::eof()) FMT_OVERRIDE {
-    if (!traits_type::eq_int_type(ch, traits_type::eof()))
-      buffer_.push_back(static_cast<Char>(ch));
-    return ch;
-  }
-
-  std::streamsize xsputn(const Char* s, std::streamsize count) FMT_OVERRIDE {
-    buffer_.append(s, s + count);
-    return count;
-  }
-};
-
-struct converter {
-  template <typename T, FMT_ENABLE_IF(is_integral<T>::value)> converter(T);
-};
-
-template <typename Char> struct test_stream : std::basic_ostream<Char> {
- private:
-  void_t<> operator<<(converter);
-};
-
-// Hide insertion operators for built-in types.
-template <typename Char, typename Traits>
-void_t<> operator<<(std::basic_ostream<Char, Traits>&, Char);
-template <typename Char, typename Traits>
-void_t<> operator<<(std::basic_ostream<Char, Traits>&, char);
-template <typename Traits>
-void_t<> operator<<(std::basic_ostream<char, Traits>&, char);
-template <typename Traits>
-void_t<> operator<<(std::basic_ostream<char, Traits>&, signed char);
-template <typename Traits>
-void_t<> operator<<(std::basic_ostream<char, Traits>&, unsigned char);
-
-// Checks if T has a user-defined operator<< (e.g. not a member of
-// std::ostream).
-template <typename T, typename Char> class is_streamable {
- private:
-  template <typename U>
-  static bool_constant<!std::is_same<decltype(std::declval<test_stream<Char>&>()
-                                              << std::declval<U>()),
-                                     void_t<>>::value>
-  test(int);
-
-  template <typename> static std::false_type test(...);
-
-  using result = decltype(test<T>(0));
-
- public:
-  is_streamable() = default;
-
-  static const bool value = result::value;
-};
-
-// Write the content of buf to os.
-template <typename Char>
-void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
-  const Char* buf_data = buf.data();
-  using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
-  unsigned_streamsize size = buf.size();
-  unsigned_streamsize max_size = to_unsigned(max_value<std::streamsize>());
-  do {
-    unsigned_streamsize n = size <= max_size ? size : max_size;
-    os.write(buf_data, static_cast<std::streamsize>(n));
-    buf_data += n;
-    size -= n;
-  } while (size != 0);
-}
-
-template <typename Char, typename T>
-void format_value(buffer<Char>& buf, const T& value,
-                  locale_ref loc = locale_ref()) {
-  formatbuf<Char> format_buf(buf);
-  std::basic_ostream<Char> output(&format_buf);
-#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
-  if (loc) output.imbue(loc.get<std::locale>());
-#endif
-  output << value;
-  output.exceptions(std::ios_base::failbit | std::ios_base::badbit);
-  buf.try_resize(buf.size());
-}
-
-// Formats an object of type T that has an overloaded ostream operator<<.
-template <typename T, typename Char>
-struct fallback_formatter<T, Char, enable_if_t<is_streamable<T, Char>::value>>
-    : private formatter<basic_string_view<Char>, Char> {
-  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
-      -> decltype(ctx.begin()) {
-    return formatter<basic_string_view<Char>, Char>::parse(ctx);
-  }
-  template <typename ParseCtx,
-            FMT_ENABLE_IF(std::is_same<
-                          ParseCtx, basic_printf_parse_context<Char>>::value)>
-  auto parse(ParseCtx& ctx) -> decltype(ctx.begin()) {
-    return ctx.begin();
-  }
-
-  template <typename OutputIt>
-  auto format(const T& value, basic_format_context<OutputIt, Char>& ctx)
-      -> OutputIt {
-    basic_memory_buffer<Char> buffer;
-    format_value(buffer, value, ctx.locale());
-    basic_string_view<Char> str(buffer.data(), buffer.size());
-    return formatter<basic_string_view<Char>, Char>::format(str, ctx);
-  }
-  template <typename OutputIt>
-  auto format(const T& value, basic_printf_context<OutputIt, Char>& ctx)
-      -> OutputIt {
-    basic_memory_buffer<Char> buffer;
-    format_value(buffer, value, ctx.locale());
-    return std::copy(buffer.begin(), buffer.end(), ctx.out());
-  }
-};
-}  // namespace detail
-
-FMT_MODULE_EXPORT
-template <typename Char>
-void vprint(std::basic_ostream<Char>& os, basic_string_view<Char> format_str,
-            basic_format_args<buffer_context<type_identity_t<Char>>> args) {
-  basic_memory_buffer<Char> buffer;
-  detail::vformat_to(buffer, format_str, args);
-  detail::write_buffer(os, buffer);
-}
-
-/**
-  \rst
-  Prints formatted data to the stream *os*.
-
-  **Example**::
-
-    fmt::print(cerr, "Don't {}!", "panic");
-  \endrst
- */
-FMT_MODULE_EXPORT
-template <typename S, typename... Args,
-          typename Char = enable_if_t<detail::is_string<S>::value, char_t<S>>>
-void print(std::basic_ostream<Char>& os, const S& format_str, Args&&... args) {
-  vprint(os, to_string_view(format_str),
-         fmt::make_args_checked<Args...>(format_str, args...));
-}
-FMT_END_NAMESPACE
-
-#endif  // FMT_OSTREAM_H_
diff --git a/contrib/fmt-8.0.1/include/fmt/ranges.h b/contrib/fmt-8.0.1/include/fmt/ranges.h
deleted file mode 100644
index f0390df211..0000000000
--- a/contrib/fmt-8.0.1/include/fmt/ranges.h
+++ /dev/null
@@ -1,468 +0,0 @@
-// Formatting library for C++ - experimental range support
-//
-// Copyright (c) 2012 - present, Victor Zverovich
-// All rights reserved.
-//
-// For the license information refer to format.h.
-//
-// Copyright (c) 2018 - present, Remotion (Igor Schulz)
-// All Rights Reserved
-// {fmt} support for ranges, containers and types tuple interface.
-
-#ifndef FMT_RANGES_H_
-#define FMT_RANGES_H_
-
-#include <initializer_list>
-#include <type_traits>
-
-#include "format.h"
-
-FMT_BEGIN_NAMESPACE
-
-template <typename Char, typename Enable = void> struct formatting_range {
-#ifdef FMT_DEPRECATED_BRACED_RANGES
-  Char prefix = '{';
-  Char postfix = '}';
-#else
-  Char prefix = '[';
-  Char postfix = ']';
-#endif
-
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    return ctx.begin();
-  }
-};
-
-template <typename Char, typename Enable = void> struct formatting_tuple {
-  Char prefix = '(';
-  Char postfix = ')';
-
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    return ctx.begin();
-  }
-};
-
-namespace detail {
-
-template <typename RangeT, typename OutputIterator>
-OutputIterator copy(const RangeT& range, OutputIterator out) {
-  for (auto it = range.begin(), end = range.end(); it != end; ++it)
-    *out++ = *it;
-  return out;
-}
-
-template <typename OutputIterator>
-OutputIterator copy(const char* str, OutputIterator out) {
-  while (*str) *out++ = *str++;
-  return out;
-}
-
-template <typename OutputIterator>
-OutputIterator copy(char ch, OutputIterator out) {
-  *out++ = ch;
-  return out;
-}
-
-template <typename OutputIterator>
-OutputIterator copy(wchar_t ch, OutputIterator out) {
-  *out++ = ch;
-  return out;
-}
-
-/// Return true value if T has std::string interface, like std::string_view.
-template <typename T> class is_std_string_like {
-  template <typename U>
-  static auto check(U* p)
-      -> decltype((void)p->find('a'), p->length(), (void)p->data(), int());
-  template <typename> static void check(...);
-
- public:
-  static FMT_CONSTEXPR_DECL const bool value =
-      is_string<T>::value || !std::is_void<decltype(check<T>(nullptr))>::value;
-};
-
-template <typename Char>
-struct is_std_string_like<fmt::basic_string_view<Char>> : std::true_type {};
-
-template <typename... Ts> struct conditional_helper {};
-
-template <typename T, typename _ = void> struct is_range_ : std::false_type {};
-
-#if !FMT_MSC_VER || FMT_MSC_VER > 1800
-
-#  define FMT_DECLTYPE_RETURN(val)  \
-    ->decltype(val) { return val; } \
-    static_assert(                  \
-        true, "")  // This makes it so that a semicolon is required after the
-                   // macro, which helps clang-format handle the formatting.
-
-// C array overload
-template <typename T, std::size_t N>
-auto range_begin(const T (&arr)[N]) -> const T* {
-  return arr;
-}
-template <typename T, std::size_t N>
-auto range_end(const T (&arr)[N]) -> const T* {
-  return arr + N;
-}
-
-template <typename T, typename Enable = void>
-struct has_member_fn_begin_end_t : std::false_type {};
-
-template <typename T>
-struct has_member_fn_begin_end_t<T, void_t<decltype(std::declval<T>().begin()),
-                                           decltype(std::declval<T>().end())>>
-    : std::true_type {};
-
-// Member function overload
-template <typename T>
-auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
-template <typename T>
-auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
-
-// ADL overload. Only participates in overload resolution if member functions
-// are not found.
-template <typename T>
-auto range_begin(T&& rng)
-    -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
-                   decltype(begin(static_cast<T&&>(rng)))> {
-  return begin(static_cast<T&&>(rng));
-}
-template <typename T>
-auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
-                                       decltype(end(static_cast<T&&>(rng)))> {
-  return end(static_cast<T&&>(rng));
-}
-
-template <typename T, typename Enable = void>
-struct has_const_begin_end : std::false_type {};
-template <typename T, typename Enable = void>
-struct has_mutable_begin_end : std::false_type {};
-
-template <typename T>
-struct has_const_begin_end<
-    T, void_t<decltype(detail::range_begin(
-                  std::declval<const remove_cvref_t<T>&>())),
-              decltype(detail::range_begin(
-                  std::declval<const remove_cvref_t<T>&>()))>>
-    : std::true_type {};
-
-template <typename T>
-struct has_mutable_begin_end<
-    T, void_t<decltype(detail::range_begin(std::declval<T>())),
-              decltype(detail::range_begin(std::declval<T>())),
-              enable_if_t<std::is_copy_constructible<T>::value>>>
-    : std::true_type {};
-
-template <typename T>
-struct is_range_<T, void>
-    : std::integral_constant<bool, (has_const_begin_end<T>::value ||
-                                    has_mutable_begin_end<T>::value)> {};
-
-template <typename T, typename Enable = void> struct range_to_view;
-template <typename T>
-struct range_to_view<T, enable_if_t<has_const_begin_end<T>::value>> {
-  struct view_t {
-    const T* m_range_ptr;
-
-    auto begin() const FMT_DECLTYPE_RETURN(detail::range_begin(*m_range_ptr));
-    auto end() const FMT_DECLTYPE_RETURN(detail::range_end(*m_range_ptr));
-  };
-  static auto view(const T& range) -> view_t { return {&range}; }
-};
-
-template <typename T>
-struct range_to_view<T, enable_if_t<!has_const_begin_end<T>::value &&
-                                    has_mutable_begin_end<T>::value>> {
-  struct view_t {
-    T m_range_copy;
-
-    auto begin() FMT_DECLTYPE_RETURN(detail::range_begin(m_range_copy));
-    auto end() FMT_DECLTYPE_RETURN(detail::range_end(m_range_copy));
-  };
-  static auto view(const T& range) -> view_t { return {range}; }
-};
-#  undef FMT_DECLTYPE_RETURN
-#endif
-
-/// tuple_size and tuple_element check.
-template <typename T> class is_tuple_like_ {
-  template <typename U>
-  static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
-  template <typename> static void check(...);
-
- public:
-  static FMT_CONSTEXPR_DECL const bool value =
-      !std::is_void<decltype(check<T>(nullptr))>::value;
-};
-
-// Check for integer_sequence
-#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VER >= 1900
-template <typename T, T... N>
-using integer_sequence = std::integer_sequence<T, N...>;
-template <size_t... N> using index_sequence = std::index_sequence<N...>;
-template <size_t N> using make_index_sequence = std::make_index_sequence<N>;
-#else
-template <typename T, T... N> struct integer_sequence {
-  using value_type = T;
-
-  static FMT_CONSTEXPR size_t size() { return sizeof...(N); }
-};
-
-template <size_t... N> using index_sequence = integer_sequence<size_t, N...>;
-
-template <typename T, size_t N, T... Ns>
-struct make_integer_sequence : make_integer_sequence<T, N - 1, N - 1, Ns...> {};
-template <typename T, T... Ns>
-struct make_integer_sequence<T, 0, Ns...> : integer_sequence<T, Ns...> {};
-
-template <size_t N>
-using make_index_sequence = make_integer_sequence<size_t, N>;
-#endif
-
-template <class Tuple, class F, size_t... Is>
-void for_each(index_sequence<Is...>, Tuple&& tup, F&& f) FMT_NOEXCEPT {
-  using std::get;
-  // using free function get<I>(T) now.
-  const int _[] = {0, ((void)f(get<Is>(tup)), 0)...};
-  (void)_;  // blocks warnings
-}
-
-template <class T>
-FMT_CONSTEXPR make_index_sequence<std::tuple_size<T>::value> get_indexes(
-    T const&) {
-  return {};
-}
-
-template <class Tuple, class F> void for_each(Tuple&& tup, F&& f) {
-  const auto indexes = get_indexes(tup);
-  for_each(indexes, std::forward<Tuple>(tup), std::forward<F>(f));
-}
-
-template <typename Range>
-using value_type =
-    remove_cvref_t<decltype(*detail::range_begin(std::declval<Range>()))>;
-
-template <typename OutputIt> OutputIt write_delimiter(OutputIt out) {
-  *out++ = ',';
-  *out++ = ' ';
-  return out;
-}
-
-template <
-    typename Char, typename OutputIt, typename Arg,
-    FMT_ENABLE_IF(is_std_string_like<typename std::decay<Arg>::type>::value)>
-OutputIt write_range_entry(OutputIt out, const Arg& v) {
-  *out++ = '"';
-  out = write<Char>(out, v);
-  *out++ = '"';
-  return out;
-}
-
-template <typename Char, typename OutputIt, typename Arg,
-          FMT_ENABLE_IF(std::is_same<Arg, Char>::value)>
-OutputIt write_range_entry(OutputIt out, const Arg v) {
-  *out++ = '\'';
-  *out++ = v;
-  *out++ = '\'';
-  return out;
-}
-
-template <
-    typename Char, typename OutputIt, typename Arg,
-    FMT_ENABLE_IF(!is_std_string_like<typename std::decay<Arg>::type>::value &&
-                  !std::is_same<Arg, Char>::value)>
-OutputIt write_range_entry(OutputIt out, const Arg& v) {
-  return write<Char>(out, v);
-}
-
-}  // namespace detail
-
-template <typename T> struct is_tuple_like {
-  static FMT_CONSTEXPR_DECL const bool value =
-      detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
-};
-
-template <typename TupleT, typename Char>
-struct formatter<TupleT, Char, enable_if_t<fmt::is_tuple_like<TupleT>::value>> {
- private:
-  // C++11 generic lambda for format()
-  template <typename FormatContext> struct format_each {
-    template <typename T> void operator()(const T& v) {
-      if (i > 0) out = detail::write_delimiter(out);
-      out = detail::write_range_entry<Char>(out, v);
-      ++i;
-    }
-    formatting_tuple<Char>& formatting;
-    size_t& i;
-    typename std::add_lvalue_reference<
-        decltype(std::declval<FormatContext>().out())>::type out;
-  };
-
- public:
-  formatting_tuple<Char> formatting;
-
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    return formatting.parse(ctx);
-  }
-
-  template <typename FormatContext = format_context>
-  auto format(const TupleT& values, FormatContext& ctx) -> decltype(ctx.out()) {
-    auto out = ctx.out();
-    size_t i = 0;
-
-    detail::copy(formatting.prefix, out);
-    detail::for_each(values, format_each<FormatContext>{formatting, i, out});
-    detail::copy(formatting.postfix, out);
-
-    return ctx.out();
-  }
-};
-
-template <typename T, typename Char> struct is_range {
-  static FMT_CONSTEXPR_DECL const bool value =
-      detail::is_range_<T>::value && !detail::is_std_string_like<T>::value &&
-      !std::is_convertible<T, std::basic_string<Char>>::value &&
-      !std::is_constructible<detail::std_string_view<Char>, T>::value;
-};
-
-template <typename T, typename Char>
-struct formatter<
-    T, Char,
-    enable_if_t<
-        fmt::is_range<T, Char>::value
-// Workaround a bug in MSVC 2017 and earlier.
-#if !FMT_MSC_VER || FMT_MSC_VER >= 1927
-        && (has_formatter<detail::value_type<T>, format_context>::value ||
-            detail::has_fallback_formatter<detail::value_type<T>, Char>::value)
-#endif
-        >> {
-  formatting_range<Char> formatting;
-
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    return formatting.parse(ctx);
-  }
-
-  template <typename FormatContext>
-  typename FormatContext::iterator format(const T& values, FormatContext& ctx) {
-    auto out = detail::copy(formatting.prefix, ctx.out());
-    size_t i = 0;
-    auto view = detail::range_to_view<T>::view(values);
-    auto it = view.begin();
-    auto end = view.end();
-    for (; it != end; ++it) {
-      if (i > 0) out = detail::write_delimiter(out);
-      out = detail::write_range_entry<Char>(out, *it);
-      ++i;
-    }
-    return detail::copy(formatting.postfix, out);
-  }
-};
-
-template <typename Char, typename... T> struct tuple_join_view : detail::view {
-  const std::tuple<T...>& tuple;
-  basic_string_view<Char> sep;
-
-  tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
-      : tuple(t), sep{s} {}
-};
-
-template <typename Char, typename... T>
-using tuple_arg_join = tuple_join_view<Char, T...>;
-
-template <typename Char, typename... T>
-struct formatter<tuple_join_view<Char, T...>, Char> {
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    return ctx.begin();
-  }
-
-  template <typename FormatContext>
-  auto format(const tuple_join_view<Char, T...>& value, FormatContext& ctx) ->
-      typename FormatContext::iterator {
-    return format(value, ctx, detail::make_index_sequence<sizeof...(T)>{});
-  }
-
- private:
-  template <typename FormatContext, size_t... N>
-  auto format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
-              detail::index_sequence<N...>) ->
-      typename FormatContext::iterator {
-    using std::get;
-    return format_args(value, ctx, get<N>(value.tuple)...);
-  }
-
-  template <typename FormatContext>
-  auto format_args(const tuple_join_view<Char, T...>&, FormatContext& ctx) ->
-      typename FormatContext::iterator {
-    // NOTE: for compilers that support C++17, this empty function instantiation
-    // can be replaced with a constexpr branch in the variadic overload.
-    return ctx.out();
-  }
-
-  template <typename FormatContext, typename Arg, typename... Args>
-  auto format_args(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
-                   const Arg& arg, const Args&... args) ->
-      typename FormatContext::iterator {
-    using base = formatter<typename std::decay<Arg>::type, Char>;
-    auto out = base().format(arg, ctx);
-    if (sizeof...(Args) > 0) {
-      out = std::copy(value.sep.begin(), value.sep.end(), out);
-      ctx.advance_to(out);
-      return format_args(value, ctx, args...);
-    }
-    return out;
-  }
-};
-
-FMT_MODULE_EXPORT_BEGIN
-
-/**
-  \rst
-  Returns an object that formats `tuple` with elements separated by `sep`.
-
-  **Example**::
-
-    std::tuple<int, char> t = {1, 'a'};
-    fmt::print("{}", fmt::join(t, ", "));
-    // Output: "1, a"
-  \endrst
- */
-template <typename... T>
-FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
-    -> tuple_join_view<char, T...> {
-  return {tuple, sep};
-}
-
-template <typename... T>
-FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple,
-                        basic_string_view<wchar_t> sep)
-    -> tuple_join_view<wchar_t, T...> {
-  return {tuple, sep};
-}
-
-/**
-  \rst
-  Returns an object that formats `initializer_list` with elements separated by
-  `sep`.
-
-  **Example**::
-
-    fmt::print("{}", fmt::join({1, 2, 3}, ", "));
-    // Output: "1, 2, 3"
-  \endrst
- */
-template <typename T>
-auto join(std::initializer_list<T> list, string_view sep)
-    -> join_view<const T*, const T*> {
-  return join(std::begin(list), std::end(list), sep);
-}
-
-FMT_MODULE_EXPORT_END
-FMT_END_NAMESPACE
-
-#endif  // FMT_RANGES_H_
diff --git a/contrib/fmt-8.1.1/.clang-format b/contrib/fmt-8.1.1/.clang-format
new file mode 100644
index 0000000000..df55d340f0
--- /dev/null
+++ b/contrib/fmt-8.1.1/.clang-format
@@ -0,0 +1,8 @@
+# Run manually to reformat a file:
+# clang-format -i --style=file <file>
+Language: Cpp
+BasedOnStyle: Google
+IndentPPDirectives: AfterHash
+IndentCaseLabels: false
+AlwaysBreakTemplateDeclarations: false
+DerivePointerAlignment: false
diff --git a/contrib/fmt-8.0.1/CMakeLists.txt b/contrib/fmt-8.1.1/CMakeLists.txt
similarity index 96%
rename from contrib/fmt-8.0.1/CMakeLists.txt
rename to contrib/fmt-8.1.1/CMakeLists.txt
index f7f3046663..3bf80f8022 100644
--- a/contrib/fmt-8.0.1/CMakeLists.txt
+++ b/contrib/fmt-8.1.1/CMakeLists.txt
@@ -81,12 +81,13 @@ option(FMT_FUZZ "Generate the fuzz target." OFF)
 option(FMT_CUDA_TEST "Generate the cuda-test target." OFF)
 option(FMT_OS "Include core requiring OS (Windows/Posix) " ON)
 option(FMT_MODULE "Build a module instead of a traditional library." OFF)
+option(FMT_SYSTEM_HEADERS "Expose headers with marking them as system." OFF)
 
 set(FMT_CAN_MODULE OFF)
 if (CMAKE_CXX_STANDARD GREATER 17 AND
     # msvc 16.10-pre4
     MSVC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 19.29.30035)
-  set(FMT_CAN_MODULE ON)
+  set(FMT_CAN_MODULE OFF)
 endif ()
 if (NOT FMT_CAN_MODULE)
   set(FMT_MODULE OFF)
@@ -96,6 +97,10 @@ if (FMT_TEST AND FMT_MODULE)
   # The tests require {fmt} to be compiled as traditional library
   message(STATUS "Testing is incompatible with build mode 'module'.")
 endif ()
+set(FMT_SYSTEM_HEADERS_ATTRIBUTE "")
+if (FMT_SYSTEM_HEADERS)
+  set(FMT_SYSTEM_HEADERS_ATTRIBUTE SYSTEM)
+endif ()
 
 # Get version from core.h
 file(READ include/fmt/core.h core_h)
@@ -151,7 +156,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
       -Wcast-align
       -Wctor-dtor-privacy -Wdisabled-optimization
       -Winvalid-pch -Woverloaded-virtual
-      -Wconversion -Wswitch-enum -Wundef
+      -Wconversion -Wundef
       -Wno-ctor-dtor-privacy -Wno-format-nonliteral)
   if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.6)
       set(PEDANTIC_COMPILE_FLAGS ${PEDANTIC_COMPILE_FLAGS}
@@ -236,7 +241,7 @@ else()
   set(FMT_SOURCES src/format.cc)
 endif ()
 
-add_library(fmt ${FMT_SOURCES} ${FMT_HEADERS})
+add_library(fmt ${FMT_SOURCES} ${FMT_HEADERS} README.rst ChangeLog.rst)
 add_library(fmt::fmt ALIAS fmt)
 
 if (HAVE_STRTOD_L)
@@ -244,7 +249,7 @@ if (HAVE_STRTOD_L)
 endif ()
 
 if (MINGW)
-  check_cxx_compiler_flag("Wa,-mbig-obj" FMT_HAS_MBIG_OBJ)
+  check_cxx_compiler_flag("-Wa,-mbig-obj" FMT_HAS_MBIG_OBJ)
   if (${FMT_HAS_MBIG_OBJ})
     target_compile_options(fmt PUBLIC "-Wa,-mbig-obj")
   endif()
@@ -262,7 +267,7 @@ endif ()
 
 target_compile_features(fmt INTERFACE ${FMT_REQUIRED_FEATURES})
 
-target_include_directories(fmt PUBLIC
+target_include_directories(fmt ${FMT_SYSTEM_HEADERS_ATTRIBUTE} PUBLIC
   $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
   $<INSTALL_INTERFACE:${FMT_INC_DIR}>)
 
@@ -298,7 +303,7 @@ add_library(fmt::fmt-header-only ALIAS fmt-header-only)
 target_compile_definitions(fmt-header-only INTERFACE FMT_HEADER_ONLY=1)
 target_compile_features(fmt-header-only INTERFACE ${FMT_REQUIRED_FEATURES})
 
-target_include_directories(fmt-header-only INTERFACE
+target_include_directories(fmt-header-only ${FMT_SYSTEM_HEADERS_ATTRIBUTE} INTERFACE
   $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
   $<INSTALL_INTERFACE:${FMT_INC_DIR}>)
 
diff --git a/contrib/fmt-8.0.1/LICENSE.rst b/contrib/fmt-8.1.1/LICENSE.rst
similarity index 100%
rename from contrib/fmt-8.0.1/LICENSE.rst
rename to contrib/fmt-8.1.1/LICENSE.rst
diff --git a/contrib/fmt-8.0.1/include/fmt/args.h b/contrib/fmt-8.1.1/include/fmt/args.h
similarity index 99%
rename from contrib/fmt-8.0.1/include/fmt/args.h
rename to contrib/fmt-8.1.1/include/fmt/args.h
index 562e8ab111..9a8e4ed2ce 100644
--- a/contrib/fmt-8.0.1/include/fmt/args.h
+++ b/contrib/fmt-8.1.1/include/fmt/args.h
@@ -143,6 +143,8 @@ class dynamic_format_arg_store
   }
 
  public:
+  constexpr dynamic_format_arg_store() = default;
+
   /**
     \rst
     Adds an argument into the dynamic store for later passing to a formatting
diff --git a/contrib/fmt-8.0.1/include/fmt/chrono.h b/contrib/fmt-8.1.1/include/fmt/chrono.h
similarity index 51%
rename from contrib/fmt-8.0.1/include/fmt/chrono.h
rename to contrib/fmt-8.1.1/include/fmt/chrono.h
index c024fd710c..682efd8d21 100644
--- a/contrib/fmt-8.0.1/include/fmt/chrono.h
+++ b/contrib/fmt-8.1.1/include/fmt/chrono.h
@@ -11,13 +11,29 @@
 #include <algorithm>
 #include <chrono>
 #include <ctime>
+#include <iterator>
 #include <locale>
-#include <sstream>
+#include <ostream>
+#include <type_traits>
 
 #include "format.h"
 
 FMT_BEGIN_NAMESPACE
 
+// Enable tzset.
+#ifndef FMT_USE_TZSET
+// UWP doesn't provide _tzset.
+#  if FMT_HAS_INCLUDE("winapifamily.h")
+#    include <winapifamily.h>
+#  endif
+#  if defined(_WIN32) && (!defined(WINAPI_FAMILY) || \
+                          (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+#    define FMT_USE_TZSET 1
+#  else
+#    define FMT_USE_TZSET 0
+#  endif
+#endif
+
 // Enable safe chrono durations, unless explicitly disabled.
 #ifndef FMT_SAFE_DURATION_CAST
 #  define FMT_SAFE_DURATION_CAST 1
@@ -44,7 +60,7 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
   static_assert(T::is_integer, "To must be integral");
 
   // A and B are both signed, or both unsigned.
-  if (F::digits <= T::digits) {
+  if (detail::const_check(F::digits <= T::digits)) {
     // From fits in To without any problem.
   } else {
     // From does not always fit in To, resort to a dynamic check.
@@ -79,14 +95,15 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
       return {};
     }
     // From is positive. Can it always fit in To?
-    if (F::digits > T::digits &&
+    if (detail::const_check(F::digits > T::digits) &&
         from > static_cast<From>(detail::max_value<To>())) {
       ec = 1;
       return {};
     }
   }
 
-  if (!F::is_signed && T::is_signed && F::digits >= T::digits &&
+  if (detail::const_check(!F::is_signed && T::is_signed &&
+                          F::digits >= T::digits) &&
       from > static_cast<From>(detail::max_value<To>())) {
     ec = 1;
     return {};
@@ -243,7 +260,7 @@ To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
   }
 
   // multiply with Factor::num without overflow or underflow
-  if (Factor::num != 1) {
+  if (detail::const_check(Factor::num != 1)) {
     constexpr auto max1 = detail::max_value<IntermediateRep>() /
                           static_cast<IntermediateRep>(Factor::num);
     if (count > max1) {
@@ -260,7 +277,7 @@ To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
   }
 
   // this can't go wrong, right? den>0 is checked earlier.
-  if (Factor::den != 1) {
+  if (detail::const_check(Factor::den != 1)) {
     using common_t = typename std::common_type<IntermediateRep, intmax_t>::type;
     count /= static_cast<common_t>(Factor::den);
   }
@@ -288,74 +305,139 @@ inline null<> localtime_s(...) { return null<>(); }
 inline null<> gmtime_r(...) { return null<>(); }
 inline null<> gmtime_s(...) { return null<>(); }
 
-inline auto do_write(const std::tm& time, const std::locale& loc, char format,
-                     char modifier) -> std::string {
-  auto&& os = std::ostringstream();
-  os.imbue(loc);
-  using iterator = std::ostreambuf_iterator<char>;
-  const auto& facet = std::use_facet<std::time_put<char, iterator>>(loc);
-  auto end = facet.put(os, os, ' ', &time, format, modifier);
-  if (end.failed()) FMT_THROW(format_error("failed to format time"));
-  auto str = os.str();
-  if (!detail::is_utf8() || loc == std::locale::classic()) return str;
+inline const std::locale& get_classic_locale() {
+  static const auto& locale = std::locale::classic();
+  return locale;
+}
+
+template <typename CodeUnit> struct codecvt_result {
+  static constexpr const size_t max_size = 32;
+  CodeUnit buf[max_size];
+  CodeUnit* end;
+};
+template <typename CodeUnit>
+constexpr const size_t codecvt_result<CodeUnit>::max_size;
+
+template <typename CodeUnit>
+void write_codecvt(codecvt_result<CodeUnit>& out, string_view in_buf,
+                   const std::locale& loc) {
+  using codecvt = std::codecvt<CodeUnit, char, std::mbstate_t>;
+#if FMT_CLANG_VERSION
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated"
+  auto& f = std::use_facet<codecvt>(loc);
+#  pragma clang diagnostic pop
+#else
+  auto& f = std::use_facet<codecvt>(loc);
+#endif
+  auto mb = std::mbstate_t();
+  const char* from_next = nullptr;
+  auto result = f.in(mb, in_buf.begin(), in_buf.end(), from_next,
+                     std::begin(out.buf), std::end(out.buf), out.end);
+  if (result != std::codecvt_base::ok)
+    FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename OutputIt>
+auto write_encoded_tm_str(OutputIt out, string_view in, const std::locale& loc)
+    -> OutputIt {
+  if (detail::is_utf8() && loc != get_classic_locale()) {
     // char16_t and char32_t codecvts are broken in MSVC (linkage errors) and
     // gcc-4.
 #if FMT_MSC_VER != 0 || \
     (defined(__GLIBCXX__) && !defined(_GLIBCXX_USE_DUAL_ABI))
-  // The _GLIBCXX_USE_DUAL_ABI macro is always defined in libstdc++ from gcc-5
-  // and newer.
-  using code_unit = wchar_t;
+    // The _GLIBCXX_USE_DUAL_ABI macro is always defined in libstdc++ from gcc-5
+    // and newer.
+    using code_unit = wchar_t;
 #else
-  using code_unit = char32_t;
+    using code_unit = char32_t;
 #endif
-  auto& f = std::use_facet<std::codecvt<code_unit, char, std::mbstate_t>>(loc);
-  auto mb = std::mbstate_t();
-  const char* from_next = nullptr;
-  code_unit* to_next = nullptr;
-  constexpr size_t buf_size = 32;
-  code_unit buf[buf_size] = {};
-  auto result = f.in(mb, str.data(), str.data() + str.size(), from_next, buf,
-                     buf + buf_size, to_next);
-  if (result != std::codecvt_base::ok)
-    FMT_THROW(format_error("failed to format time"));
-  str.clear();
-  for (code_unit* p = buf; p != to_next; ++p) {
-    uint32_t c = static_cast<uint32_t>(*p);
-    if (sizeof(code_unit) == 2 && c >= 0xd800 && c <= 0xdfff) {
-      // surrogate pair
-      ++p;
-      if (p == to_next || (c & 0xfc00) != 0xd800 || (*p & 0xfc00) != 0xdc00) {
+
+    using unit_t = codecvt_result<code_unit>;
+    unit_t unit;
+    write_codecvt(unit, in, loc);
+    // In UTF-8 is used one to four one-byte code units.
+    auto&& buf = basic_memory_buffer<char, unit_t::max_size * 4>();
+    for (code_unit* p = unit.buf; p != unit.end; ++p) {
+      uint32_t c = static_cast<uint32_t>(*p);
+      if (sizeof(code_unit) == 2 && c >= 0xd800 && c <= 0xdfff) {
+        // surrogate pair
+        ++p;
+        if (p == unit.end || (c & 0xfc00) != 0xd800 ||
+            (*p & 0xfc00) != 0xdc00) {
+          FMT_THROW(format_error("failed to format time"));
+        }
+        c = (c << 10) + static_cast<uint32_t>(*p) - 0x35fdc00;
+      }
+      if (c < 0x80) {
+        buf.push_back(static_cast<char>(c));
+      } else if (c < 0x800) {
+        buf.push_back(static_cast<char>(0xc0 | (c >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) {
+        buf.push_back(static_cast<char>(0xe0 | (c >> 12)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else if (c >= 0x10000 && c <= 0x10ffff) {
+        buf.push_back(static_cast<char>(0xf0 | (c >> 18)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0x3ffff) >> 12)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else {
         FMT_THROW(format_error("failed to format time"));
       }
-      c = (c << 10) + static_cast<uint32_t>(*p) - 0x35fdc00;
-    }
-    if (c < 0x80) {
-      str.push_back(static_cast<char>(c));
-    } else if (c < 0x800) {
-      str.push_back(static_cast<char>(0xc0 | (c >> 6)));
-      str.push_back(static_cast<char>(0x80 | (c & 0x3f)));
-    } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) {
-      str.push_back(static_cast<char>(0xe0 | (c >> 12)));
-      str.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
-      str.push_back(static_cast<char>(0x80 | (c & 0x3f)));
-    } else if (c >= 0x10000 && c <= 0x10ffff) {
-      str.push_back(static_cast<char>(0xf0 | (c >> 18)));
-      str.push_back(static_cast<char>(0x80 | ((c & 0x3ffff) >> 12)));
-      str.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
-      str.push_back(static_cast<char>(0x80 | (c & 0x3f)));
-    } else {
-      FMT_THROW(format_error("failed to format time"));
     }
+    return copy_str<char>(buf.data(), buf.data() + buf.size(), out);
   }
-  return str;
+  return copy_str<char>(in.data(), in.data() + in.size(), out);
 }
 
-template <typename OutputIt>
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+    -> OutputIt {
+  codecvt_result<Char> unit;
+  write_codecvt(unit, sv, loc);
+  return copy_str<Char>(unit.buf, unit.end, out);
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+    -> OutputIt {
+  return write_encoded_tm_str(out, sv, loc);
+}
+
+template <typename Char>
+inline void do_write(buffer<Char>& buf, const std::tm& time,
+                     const std::locale& loc, char format, char modifier) {
+  auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+  auto&& os = std::basic_ostream<Char>(&format_buf);
+  os.imbue(loc);
+  using iterator = std::ostreambuf_iterator<Char>;
+  const auto& facet = std::use_facet<std::time_put<Char, iterator>>(loc);
+  auto end = facet.put(os, os, Char(' '), &time, format, modifier);
+  if (end.failed()) FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
 auto write(OutputIt out, const std::tm& time, const std::locale& loc,
            char format, char modifier = 0) -> OutputIt {
-  auto str = do_write(time, loc, format, modifier);
-  return std::copy(str.begin(), str.end(), out);
+  auto&& buf = get_buffer<Char>(out);
+  do_write<Char>(buf, time, loc, format, modifier);
+  return buf.out();
 }
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+           char format, char modifier = 0) -> OutputIt {
+  auto&& buf = basic_memory_buffer<Char>();
+  do_write<char>(buf, time, loc, format, modifier);
+  return write_encoded_tm_str(out, string_view(buf.data(), buf.size()), loc);
+}
+
 }  // namespace detail
 
 FMT_MODULE_EXPORT_BEGIN
@@ -453,103 +535,40 @@ inline std::tm gmtime(
 
 FMT_BEGIN_DETAIL_NAMESPACE
 
-inline size_t strftime(char* str, size_t count, const char* format,
-                       const std::tm* time) {
-  // Assign to a pointer to suppress GCCs -Wformat-nonliteral
-  // First assign the nullptr to suppress -Wsuggest-attribute=format
-  std::size_t (*strftime)(char*, std::size_t, const char*, const std::tm*) =
-      nullptr;
-  strftime = std::strftime;
-  return strftime(str, count, format, time);
+// Writes two-digit numbers a, b and c separated by sep to buf.
+// The method by Pavel Novikov based on
+// https://johnnylee-sde.github.io/Fast-unsigned-integer-to-time-string/.
+inline void write_digit2_separated(char* buf, unsigned a, unsigned b,
+                                   unsigned c, char sep) {
+  unsigned long long digits =
+      a | (b << 24) | (static_cast<unsigned long long>(c) << 48);
+  // Convert each value to BCD.
+  // We have x = a * 10 + b and we want to convert it to BCD y = a * 16 + b.
+  // The difference is
+  //   y - x = a * 6
+  // a can be found from x:
+  //   a = floor(x / 10)
+  // then
+  //   y = x + a * 6 = x + floor(x / 10) * 6
+  // floor(x / 10) is (x * 205) >> 11 (needs 16 bits).
+  digits += (((digits * 205) >> 11) & 0x000f00000f00000f) * 6;
+  // Put low nibbles to high bytes and high nibbles to low bytes.
+  digits = ((digits & 0x00f00000f00000f0) >> 4) |
+           ((digits & 0x000f00000f00000f) << 8);
+  auto usep = static_cast<unsigned long long>(sep);
+  // Add ASCII '0' to each digit byte and insert separators.
+  digits |= 0x3030003030003030 | (usep << 16) | (usep << 40);
+
+  constexpr const size_t len = 8;
+  if (const_check(is_big_endian())) {
+    char tmp[len];
+    memcpy(tmp, &digits, len);
+    std::reverse_copy(tmp, tmp + len, buf);
+  } else {
+    memcpy(buf, &digits, len);
+  }
 }
 
-inline size_t strftime(wchar_t* str, size_t count, const wchar_t* format,
-                       const std::tm* time) {
-  // See above
-  std::size_t (*wcsftime)(wchar_t*, std::size_t, const wchar_t*,
-                          const std::tm*) = nullptr;
-  wcsftime = std::wcsftime;
-  return wcsftime(str, count, format, time);
-}
-
-FMT_END_DETAIL_NAMESPACE
-
-template <typename Char, typename Duration>
-struct formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
-                 Char> : formatter<std::tm, Char> {
-  FMT_CONSTEXPR formatter() {
-    this->specs = {default_specs, sizeof(default_specs) / sizeof(Char)};
-  }
-
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    auto it = ctx.begin();
-    if (it != ctx.end() && *it == ':') ++it;
-    auto end = it;
-    while (end != ctx.end() && *end != '}') ++end;
-    if (end != it) this->specs = {it, detail::to_unsigned(end - it)};
-    return end;
-  }
-
-  template <typename FormatContext>
-  auto format(std::chrono::time_point<std::chrono::system_clock> val,
-              FormatContext& ctx) -> decltype(ctx.out()) {
-    std::tm time = localtime(val);
-    return formatter<std::tm, Char>::format(time, ctx);
-  }
-
-  static constexpr Char default_specs[] = {'%', 'Y', '-', '%', 'm', '-',
-                                           '%', 'd', ' ', '%', 'H', ':',
-                                           '%', 'M', ':', '%', 'S'};
-};
-
-template <typename Char, typename Duration>
-constexpr Char
-    formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
-              Char>::default_specs[];
-
-template <typename Char> struct formatter<std::tm, Char> {
-  template <typename ParseContext>
-  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
-    auto it = ctx.begin();
-    if (it != ctx.end() && *it == ':') ++it;
-    auto end = it;
-    while (end != ctx.end() && *end != '}') ++end;
-    specs = {it, detail::to_unsigned(end - it)};
-    return end;
-  }
-
-  template <typename FormatContext>
-  auto format(const std::tm& tm, FormatContext& ctx) const
-      -> decltype(ctx.out()) {
-    basic_memory_buffer<Char> tm_format;
-    tm_format.append(specs.begin(), specs.end());
-    // By appending an extra space we can distinguish an empty result that
-    // indicates insufficient buffer size from a guaranteed non-empty result
-    // https://github.com/fmtlib/fmt/issues/2238
-    tm_format.push_back(' ');
-    tm_format.push_back('\0');
-    basic_memory_buffer<Char> buf;
-    size_t start = buf.size();
-    for (;;) {
-      size_t size = buf.capacity() - start;
-      size_t count = detail::strftime(&buf[start], size, &tm_format[0], &tm);
-      if (count != 0) {
-        buf.resize(start + count);
-        break;
-      }
-      const size_t MIN_GROWTH = 10;
-      buf.reserve(buf.capacity() + (size > MIN_GROWTH ? size : MIN_GROWTH));
-    }
-    // Remove the extra space.
-    return std::copy(buf.begin(), buf.end() - 1, ctx.out());
-  }
-
-  basic_string_view<Char> specs;
-};
-
-FMT_BEGIN_DETAIL_NAMESPACE
-
 template <typename Period> FMT_CONSTEXPR inline const char* get_units() {
   if (std::is_same<Period, std::atto>::value) return "as";
   if (std::is_same<Period, std::femto>::value) return "fs";
@@ -610,6 +629,22 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
       handler.on_text(tab, tab + 1);
       break;
     }
+    // Year:
+    case 'Y':
+      handler.on_year(numeric_system::standard);
+      break;
+    case 'y':
+      handler.on_short_year(numeric_system::standard);
+      break;
+    case 'C':
+      handler.on_century(numeric_system::standard);
+      break;
+    case 'G':
+      handler.on_iso_week_based_year();
+      break;
+    case 'g':
+      handler.on_iso_week_based_short_year();
+      break;
     // Day of the week:
     case 'a':
       handler.on_abbr_weekday();
@@ -625,11 +660,34 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
       break;
     // Month:
     case 'b':
+    case 'h':
       handler.on_abbr_month();
       break;
     case 'B':
       handler.on_full_month();
       break;
+    case 'm':
+      handler.on_dec_month(numeric_system::standard);
+      break;
+    // Day of the year/month:
+    case 'U':
+      handler.on_dec0_week_of_year(numeric_system::standard);
+      break;
+    case 'W':
+      handler.on_dec1_week_of_year(numeric_system::standard);
+      break;
+    case 'V':
+      handler.on_iso_week_of_year(numeric_system::standard);
+      break;
+    case 'j':
+      handler.on_day_of_year();
+      break;
+    case 'd':
+      handler.on_day_of_month(numeric_system::standard);
+      break;
+    case 'e':
+      handler.on_day_of_month_space(numeric_system::standard);
+      break;
     // Hour, minute, second:
     case 'H':
       handler.on_24_hour(numeric_system::standard);
@@ -688,6 +746,15 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
       if (ptr == end) FMT_THROW(format_error("invalid format"));
       c = *ptr++;
       switch (c) {
+      case 'Y':
+        handler.on_year(numeric_system::alternative);
+        break;
+      case 'y':
+        handler.on_offset_year();
+        break;
+      case 'C':
+        handler.on_century(numeric_system::alternative);
+        break;
       case 'c':
         handler.on_datetime(numeric_system::alternative);
         break;
@@ -706,6 +773,27 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
       if (ptr == end) FMT_THROW(format_error("invalid format"));
       c = *ptr++;
       switch (c) {
+      case 'y':
+        handler.on_short_year(numeric_system::alternative);
+        break;
+      case 'm':
+        handler.on_dec_month(numeric_system::alternative);
+        break;
+      case 'U':
+        handler.on_dec0_week_of_year(numeric_system::alternative);
+        break;
+      case 'W':
+        handler.on_dec1_week_of_year(numeric_system::alternative);
+        break;
+      case 'V':
+        handler.on_iso_week_of_year(numeric_system::alternative);
+        break;
+      case 'd':
+        handler.on_day_of_month(numeric_system::alternative);
+        break;
+      case 'e':
+        handler.on_day_of_month_space(numeric_system::alternative);
+        break;
       case 'w':
         handler.on_dec0_weekday(numeric_system::alternative);
         break;
@@ -741,12 +829,25 @@ template <typename Derived> struct null_chrono_spec_handler {
   FMT_CONSTEXPR void unsupported() {
     static_cast<Derived*>(this)->unsupported();
   }
+  FMT_CONSTEXPR void on_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_short_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_offset_year() { unsupported(); }
+  FMT_CONSTEXPR void on_century(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_iso_week_based_year() { unsupported(); }
+  FMT_CONSTEXPR void on_iso_week_based_short_year() { unsupported(); }
   FMT_CONSTEXPR void on_abbr_weekday() { unsupported(); }
   FMT_CONSTEXPR void on_full_weekday() { unsupported(); }
   FMT_CONSTEXPR void on_dec0_weekday(numeric_system) { unsupported(); }
   FMT_CONSTEXPR void on_dec1_weekday(numeric_system) { unsupported(); }
   FMT_CONSTEXPR void on_abbr_month() { unsupported(); }
   FMT_CONSTEXPR void on_full_month() { unsupported(); }
+  FMT_CONSTEXPR void on_dec_month(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_day_of_year() { unsupported(); }
+  FMT_CONSTEXPR void on_day_of_month(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_day_of_month_space(numeric_system) { unsupported(); }
   FMT_CONSTEXPR void on_24_hour(numeric_system) { unsupported(); }
   FMT_CONSTEXPR void on_12_hour(numeric_system) { unsupported(); }
   FMT_CONSTEXPR void on_minute(numeric_system) { unsupported(); }
@@ -766,6 +867,509 @@ template <typename Derived> struct null_chrono_spec_handler {
   FMT_CONSTEXPR void on_tz_name() { unsupported(); }
 };
 
+struct tm_format_checker : null_chrono_spec_handler<tm_format_checker> {
+  FMT_NORETURN void unsupported() { FMT_THROW(format_error("no format")); }
+
+  template <typename Char>
+  FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+  FMT_CONSTEXPR void on_year(numeric_system) {}
+  FMT_CONSTEXPR void on_short_year(numeric_system) {}
+  FMT_CONSTEXPR void on_offset_year() {}
+  FMT_CONSTEXPR void on_century(numeric_system) {}
+  FMT_CONSTEXPR void on_iso_week_based_year() {}
+  FMT_CONSTEXPR void on_iso_week_based_short_year() {}
+  FMT_CONSTEXPR void on_abbr_weekday() {}
+  FMT_CONSTEXPR void on_full_weekday() {}
+  FMT_CONSTEXPR void on_dec0_weekday(numeric_system) {}
+  FMT_CONSTEXPR void on_dec1_weekday(numeric_system) {}
+  FMT_CONSTEXPR void on_abbr_month() {}
+  FMT_CONSTEXPR void on_full_month() {}
+  FMT_CONSTEXPR void on_dec_month(numeric_system) {}
+  FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) {}
+  FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) {}
+  FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) {}
+  FMT_CONSTEXPR void on_day_of_year() {}
+  FMT_CONSTEXPR void on_day_of_month(numeric_system) {}
+  FMT_CONSTEXPR void on_day_of_month_space(numeric_system) {}
+  FMT_CONSTEXPR void on_24_hour(numeric_system) {}
+  FMT_CONSTEXPR void on_12_hour(numeric_system) {}
+  FMT_CONSTEXPR void on_minute(numeric_system) {}
+  FMT_CONSTEXPR void on_second(numeric_system) {}
+  FMT_CONSTEXPR void on_datetime(numeric_system) {}
+  FMT_CONSTEXPR void on_loc_date(numeric_system) {}
+  FMT_CONSTEXPR void on_loc_time(numeric_system) {}
+  FMT_CONSTEXPR void on_us_date() {}
+  FMT_CONSTEXPR void on_iso_date() {}
+  FMT_CONSTEXPR void on_12_hour_time() {}
+  FMT_CONSTEXPR void on_24_hour_time() {}
+  FMT_CONSTEXPR void on_iso_time() {}
+  FMT_CONSTEXPR void on_am_pm() {}
+  FMT_CONSTEXPR void on_utc_offset() {}
+  FMT_CONSTEXPR void on_tz_name() {}
+};
+
+inline const char* tm_wday_full_name(int wday) {
+  static constexpr const char* full_name_list[] = {
+      "Sunday",   "Monday", "Tuesday", "Wednesday",
+      "Thursday", "Friday", "Saturday"};
+  return wday >= 0 && wday <= 6 ? full_name_list[wday] : "?";
+}
+inline const char* tm_wday_short_name(int wday) {
+  static constexpr const char* short_name_list[] = {"Sun", "Mon", "Tue", "Wed",
+                                                    "Thu", "Fri", "Sat"};
+  return wday >= 0 && wday <= 6 ? short_name_list[wday] : "???";
+}
+
+inline const char* tm_mon_full_name(int mon) {
+  static constexpr const char* full_name_list[] = {
+      "January", "February", "March",     "April",   "May",      "June",
+      "July",    "August",   "September", "October", "November", "December"};
+  return mon >= 0 && mon <= 11 ? full_name_list[mon] : "?";
+}
+inline const char* tm_mon_short_name(int mon) {
+  static constexpr const char* short_name_list[] = {
+      "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+      "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+  };
+  return mon >= 0 && mon <= 11 ? short_name_list[mon] : "???";
+}
+
+template <typename T, typename = void>
+struct has_member_data_tm_gmtoff : std::false_type {};
+template <typename T>
+struct has_member_data_tm_gmtoff<T, void_t<decltype(T::tm_gmtoff)>>
+    : std::true_type {};
+
+template <typename T, typename = void>
+struct has_member_data_tm_zone : std::false_type {};
+template <typename T>
+struct has_member_data_tm_zone<T, void_t<decltype(T::tm_zone)>>
+    : std::true_type {};
+
+#if FMT_USE_TZSET
+inline void tzset_once() {
+  static bool init = []() -> bool {
+    _tzset();
+    return true;
+  }();
+  ignore_unused(init);
+}
+#endif
+
+template <typename OutputIt, typename Char> class tm_writer {
+ private:
+  static constexpr int days_per_week = 7;
+
+  const std::locale& loc_;
+  const bool is_classic_;
+  OutputIt out_;
+  const std::tm& tm_;
+
+  auto tm_sec() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_sec >= 0 && tm_.tm_sec <= 61, "");
+    return tm_.tm_sec;
+  }
+  auto tm_min() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_min >= 0 && tm_.tm_min <= 59, "");
+    return tm_.tm_min;
+  }
+  auto tm_hour() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_hour >= 0 && tm_.tm_hour <= 23, "");
+    return tm_.tm_hour;
+  }
+  auto tm_mday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_mday >= 1 && tm_.tm_mday <= 31, "");
+    return tm_.tm_mday;
+  }
+  auto tm_mon() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_mon >= 0 && tm_.tm_mon <= 11, "");
+    return tm_.tm_mon;
+  }
+  auto tm_year() const noexcept -> long long { return 1900ll + tm_.tm_year; }
+  auto tm_wday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_wday >= 0 && tm_.tm_wday <= 6, "");
+    return tm_.tm_wday;
+  }
+  auto tm_yday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_yday >= 0 && tm_.tm_yday <= 365, "");
+    return tm_.tm_yday;
+  }
+
+  auto tm_hour12() const noexcept -> int {
+    const auto h = tm_hour();
+    const auto z = h < 12 ? h : h - 12;
+    return z == 0 ? 12 : z;
+  }
+
+  // POSIX and the C Standard are unclear or inconsistent about what %C and %y
+  // do if the year is negative or exceeds 9999. Use the convention that %C
+  // concatenated with %y yields the same output as %Y, and that %Y contains at
+  // least 4 characters, with more only if necessary.
+  auto split_year_lower(long long year) const noexcept -> int {
+    auto l = year % 100;
+    if (l < 0) l = -l;  // l in [0, 99]
+    return static_cast<int>(l);
+  }
+
+  // Algorithm:
+  // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_the_week_number_from_a_month_and_day_of_the_month_or_ordinal_date
+  auto iso_year_weeks(long long curr_year) const noexcept -> int {
+    const auto prev_year = curr_year - 1;
+    const auto curr_p =
+        (curr_year + curr_year / 4 - curr_year / 100 + curr_year / 400) %
+        days_per_week;
+    const auto prev_p =
+        (prev_year + prev_year / 4 - prev_year / 100 + prev_year / 400) %
+        days_per_week;
+    return 52 + ((curr_p == 4 || prev_p == 3) ? 1 : 0);
+  }
+  auto iso_week_num(int tm_yday, int tm_wday) const noexcept -> int {
+    return (tm_yday + 11 - (tm_wday == 0 ? days_per_week : tm_wday)) /
+           days_per_week;
+  }
+  auto tm_iso_week_year() const noexcept -> long long {
+    const auto year = tm_year();
+    const auto w = iso_week_num(tm_yday(), tm_wday());
+    if (w < 1) return year - 1;
+    if (w > iso_year_weeks(year)) return year + 1;
+    return year;
+  }
+  auto tm_iso_week_of_year() const noexcept -> int {
+    const auto year = tm_year();
+    const auto w = iso_week_num(tm_yday(), tm_wday());
+    if (w < 1) return iso_year_weeks(year - 1);
+    if (w > iso_year_weeks(year)) return 1;
+    return w;
+  }
+
+  void write1(int value) {
+    *out_++ = static_cast<char>('0' + to_unsigned(value) % 10);
+  }
+  void write2(int value) {
+    const char* d = digits2(to_unsigned(value) % 100);
+    *out_++ = *d++;
+    *out_++ = *d;
+  }
+
+  void write_year_extended(long long year) {
+    // At least 4 characters.
+    int width = 4;
+    if (year < 0) {
+      *out_++ = '-';
+      year = 0 - year;
+      --width;
+    }
+    uint32_or_64_or_128_t<long long> n = to_unsigned(year);
+    const int num_digits = count_digits(n);
+    if (width > num_digits) out_ = std::fill_n(out_, width - num_digits, '0');
+    out_ = format_decimal<Char>(out_, n, num_digits).end;
+  }
+  void write_year(long long year) {
+    if (year >= 0 && year < 10000) {
+      write2(static_cast<int>(year / 100));
+      write2(static_cast<int>(year % 100));
+    } else {
+      write_year_extended(year);
+    }
+  }
+
+  void write_utc_offset(long offset) {
+    if (offset < 0) {
+      *out_++ = '-';
+      offset = -offset;
+    } else {
+      *out_++ = '+';
+    }
+    offset /= 60;
+    write2(static_cast<int>(offset / 60));
+    write2(static_cast<int>(offset % 60));
+  }
+  template <typename T, FMT_ENABLE_IF(has_member_data_tm_gmtoff<T>::value)>
+  void format_utc_offset_impl(const T& tm) {
+    write_utc_offset(tm.tm_gmtoff);
+  }
+  template <typename T, FMT_ENABLE_IF(!has_member_data_tm_gmtoff<T>::value)>
+  void format_utc_offset_impl(const T& tm) {
+#if defined(_WIN32) && defined(_UCRT)
+#  if FMT_USE_TZSET
+    tzset_once();
+#  endif
+    long offset = 0;
+    _get_timezone(&offset);
+    if (tm.tm_isdst) {
+      long dstbias = 0;
+      _get_dstbias(&dstbias);
+      offset += dstbias;
+    }
+    write_utc_offset(-offset);
+#else
+    ignore_unused(tm);
+    format_localized('z');
+#endif
+  }
+
+  template <typename T, FMT_ENABLE_IF(has_member_data_tm_zone<T>::value)>
+  void format_tz_name_impl(const T& tm) {
+    if (is_classic_)
+      out_ = write_tm_str<Char>(out_, tm.tm_zone, loc_);
+    else
+      format_localized('Z');
+  }
+  template <typename T, FMT_ENABLE_IF(!has_member_data_tm_zone<T>::value)>
+  void format_tz_name_impl(const T&) {
+    format_localized('Z');
+  }
+
+  void format_localized(char format, char modifier = 0) {
+    out_ = write<Char>(out_, tm_, loc_, format, modifier);
+  }
+
+ public:
+  tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm)
+      : loc_(loc),
+        is_classic_(loc_ == get_classic_locale()),
+        out_(out),
+        tm_(tm) {}
+
+  OutputIt out() const { return out_; }
+
+  FMT_CONSTEXPR void on_text(const Char* begin, const Char* end) {
+    out_ = copy_str<Char>(begin, end, out_);
+  }
+
+  void on_abbr_weekday() {
+    if (is_classic_)
+      out_ = write(out_, tm_wday_short_name(tm_wday()));
+    else
+      format_localized('a');
+  }
+  void on_full_weekday() {
+    if (is_classic_)
+      out_ = write(out_, tm_wday_full_name(tm_wday()));
+    else
+      format_localized('A');
+  }
+  void on_dec0_weekday(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write1(tm_wday());
+    format_localized('w', 'O');
+  }
+  void on_dec1_weekday(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto wday = tm_wday();
+      write1(wday == 0 ? days_per_week : wday);
+    } else {
+      format_localized('u', 'O');
+    }
+  }
+
+  void on_abbr_month() {
+    if (is_classic_)
+      out_ = write(out_, tm_mon_short_name(tm_mon()));
+    else
+      format_localized('b');
+  }
+  void on_full_month() {
+    if (is_classic_)
+      out_ = write(out_, tm_mon_full_name(tm_mon()));
+    else
+      format_localized('B');
+  }
+
+  void on_datetime(numeric_system ns) {
+    if (is_classic_) {
+      on_abbr_weekday();
+      *out_++ = ' ';
+      on_abbr_month();
+      *out_++ = ' ';
+      on_day_of_month_space(numeric_system::standard);
+      *out_++ = ' ';
+      on_iso_time();
+      *out_++ = ' ';
+      on_year(numeric_system::standard);
+    } else {
+      format_localized('c', ns == numeric_system::standard ? '\0' : 'E');
+    }
+  }
+  void on_loc_date(numeric_system ns) {
+    if (is_classic_)
+      on_us_date();
+    else
+      format_localized('x', ns == numeric_system::standard ? '\0' : 'E');
+  }
+  void on_loc_time(numeric_system ns) {
+    if (is_classic_)
+      on_iso_time();
+    else
+      format_localized('X', ns == numeric_system::standard ? '\0' : 'E');
+  }
+  void on_us_date() {
+    char buf[8];
+    write_digit2_separated(buf, to_unsigned(tm_mon() + 1),
+                           to_unsigned(tm_mday()),
+                           to_unsigned(split_year_lower(tm_year())), '/');
+    out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+  }
+  void on_iso_date() {
+    auto year = tm_year();
+    char buf[10];
+    size_t offset = 0;
+    if (year >= 0 && year < 10000) {
+      copy2(buf, digits2(to_unsigned(year / 100)));
+    } else {
+      offset = 4;
+      write_year_extended(year);
+      year = 0;
+    }
+    write_digit2_separated(buf + 2, static_cast<unsigned>(year % 100),
+                           to_unsigned(tm_mon() + 1), to_unsigned(tm_mday()),
+                           '-');
+    out_ = copy_str<Char>(std::begin(buf) + offset, std::end(buf), out_);
+  }
+
+  void on_utc_offset() { format_utc_offset_impl(tm_); }
+  void on_tz_name() { format_tz_name_impl(tm_); }
+
+  void on_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write_year(tm_year());
+    format_localized('Y', 'E');
+  }
+  void on_short_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(split_year_lower(tm_year()));
+    format_localized('y', 'O');
+  }
+  void on_offset_year() {
+    if (is_classic_) return write2(split_year_lower(tm_year()));
+    format_localized('y', 'E');
+  }
+
+  void on_century(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto year = tm_year();
+      auto upper = year / 100;
+      if (year >= -99 && year < 0) {
+        // Zero upper on negative year.
+        *out_++ = '-';
+        *out_++ = '0';
+      } else if (upper >= 0 && upper < 100) {
+        write2(static_cast<int>(upper));
+      } else {
+        out_ = write<Char>(out_, upper);
+      }
+    } else {
+      format_localized('C', 'E');
+    }
+  }
+
+  void on_dec_month(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_mon() + 1);
+    format_localized('m', 'O');
+  }
+
+  void on_dec0_week_of_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2((tm_yday() + days_per_week - tm_wday()) / days_per_week);
+    format_localized('U', 'O');
+  }
+  void on_dec1_week_of_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto wday = tm_wday();
+      write2((tm_yday() + days_per_week -
+              (wday == 0 ? (days_per_week - 1) : (wday - 1))) /
+             days_per_week);
+    } else {
+      format_localized('W', 'O');
+    }
+  }
+  void on_iso_week_of_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_iso_week_of_year());
+    format_localized('V', 'O');
+  }
+
+  void on_iso_week_based_year() { write_year(tm_iso_week_year()); }
+  void on_iso_week_based_short_year() {
+    write2(split_year_lower(tm_iso_week_year()));
+  }
+
+  void on_day_of_year() {
+    auto yday = tm_yday() + 1;
+    write1(yday / 100);
+    write2(yday % 100);
+  }
+  void on_day_of_month(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write2(tm_mday());
+    format_localized('d', 'O');
+  }
+  void on_day_of_month_space(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto mday = to_unsigned(tm_mday()) % 100;
+      const char* d2 = digits2(mday);
+      *out_++ = mday < 10 ? ' ' : d2[0];
+      *out_++ = d2[1];
+    } else {
+      format_localized('e', 'O');
+    }
+  }
+
+  void on_24_hour(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write2(tm_hour());
+    format_localized('H', 'O');
+  }
+  void on_12_hour(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_hour12());
+    format_localized('I', 'O');
+  }
+  void on_minute(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write2(tm_min());
+    format_localized('M', 'O');
+  }
+  void on_second(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write2(tm_sec());
+    format_localized('S', 'O');
+  }
+
+  void on_12_hour_time() {
+    if (is_classic_) {
+      char buf[8];
+      write_digit2_separated(buf, to_unsigned(tm_hour12()),
+                             to_unsigned(tm_min()), to_unsigned(tm_sec()), ':');
+      out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+      *out_++ = ' ';
+      on_am_pm();
+    } else {
+      format_localized('r');
+    }
+  }
+  void on_24_hour_time() {
+    write2(tm_hour());
+    *out_++ = ':';
+    write2(tm_min());
+  }
+  void on_iso_time() {
+    char buf[8];
+    write_digit2_separated(buf, to_unsigned(tm_hour()), to_unsigned(tm_min()),
+                           to_unsigned(tm_sec()), ':');
+    out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+  }
+
+  void on_am_pm() {
+    if (is_classic_) {
+      *out_++ = tm_hour() < 12 ? 'A' : 'P';
+      *out_++ = 'M';
+    } else {
+      format_localized('p');
+    }
+  }
+
+  // These apply to chrono durations but not tm.
+  void on_duration_value() {}
+  void on_duration_unit() {}
+};
+
 struct chrono_format_checker : null_chrono_spec_handler<chrono_format_checker> {
   FMT_NORETURN void unsupported() { FMT_THROW(format_error("no date")); }
 
@@ -796,26 +1400,20 @@ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
 inline bool isfinite(T) {
   return true;
 }
-template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
-inline bool isfinite(T value) {
-  return std::isfinite(value);
-}
 
-// Converts value to int and checks that it's in the range [0, upper).
-template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
-inline int to_nonnegative_int(T value, int upper) {
+// Converts value to Int and checks that it's in the range [0, upper).
+template <typename T, typename Int, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
   FMT_ASSERT(value >= 0 && to_unsigned(value) <= to_unsigned(upper),
              "invalid value");
   (void)upper;
-  return static_cast<int>(value);
+  return static_cast<Int>(value);
 }
-template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
-inline int to_nonnegative_int(T value, int upper) {
-  FMT_ASSERT(
-      std::isnan(value) || (value >= 0 && value <= static_cast<T>(upper)),
-      "invalid value");
-  (void)upper;
-  return static_cast<int>(value);
+template <typename T, typename Int, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
+  if (value < 0 || value > static_cast<T>(upper))
+    FMT_THROW(format_error("invalid value"));
+  return static_cast<Int>(value);
 }
 
 template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
@@ -872,15 +1470,37 @@ inline std::chrono::duration<Rep, std::milli> get_milliseconds(
 #endif
 }
 
-template <typename Rep, typename Period,
-          FMT_ENABLE_IF(std::is_floating_point<Rep>::value)>
-inline std::chrono::duration<Rep, std::milli> get_milliseconds(
+// Returns the number of fractional digits in the range [0, 18] according to the
+// C++20 spec. If more than 18 fractional digits are required then returns 6 for
+// microseconds precision.
+constexpr int count_fractional_digits(long long num, long long den, int n = 0) {
+  return num % den == 0
+             ? n
+             : (n > 18 ? 6 : count_fractional_digits(num * 10, den, n + 1));
+}
+
+constexpr long long pow10(std::uint32_t n) {
+  return n == 0 ? 1 : 10 * pow10(n - 1);
+}
+
+template <class Rep, class Period,
+          FMT_ENABLE_IF(std::numeric_limits<Rep>::is_signed)>
+constexpr std::chrono::duration<Rep, Period> abs(
     std::chrono::duration<Rep, Period> d) {
-  using common_type = typename std::common_type<Rep, std::intmax_t>::type;
-  auto ms = mod(d.count() * static_cast<common_type>(Period::num) /
-                    static_cast<common_type>(Period::den) * 1000,
-                1000);
-  return std::chrono::duration<Rep, std::milli>(static_cast<Rep>(ms));
+  // We need to compare the duration using the count() method directly
+  // due to a compiler bug in clang-11 regarding the spaceship operator,
+  // when -Wzero-as-null-pointer-constant is enabled.
+  // In clang-12 the bug has been fixed. See
+  // https://bugs.llvm.org/show_bug.cgi?id=46235 and the reproducible example:
+  // https://www.godbolt.org/z/Knbb5joYx.
+  return d.count() >= d.zero().count() ? d : -d;
+}
+
+template <class Rep, class Period,
+          FMT_ENABLE_IF(!std::numeric_limits<Rep>::is_signed)>
+constexpr std::chrono::duration<Rep, Period> abs(
+    std::chrono::duration<Rep, Period> d) {
+  return d;
 }
 
 template <typename Char, typename Rep, typename OutputIt,
@@ -894,7 +1514,8 @@ template <typename Char, typename Rep, typename OutputIt,
 OutputIt format_duration_value(OutputIt out, Rep val, int precision) {
   auto specs = basic_format_specs<Char>();
   specs.precision = precision;
-  specs.type = precision > 0 ? 'f' : 'g';
+  specs.type = precision >= 0 ? presentation_type::fixed_lower
+                              : presentation_type::general_lower;
   return write<Char>(out, val, specs);
 }
 
@@ -926,6 +1547,26 @@ OutputIt format_duration_unit(OutputIt out) {
   return out;
 }
 
+class get_locale {
+ private:
+  union {
+    std::locale locale_;
+  };
+  bool has_locale_ = false;
+
+ public:
+  get_locale(bool localized, locale_ref loc) : has_locale_(localized) {
+    if (localized)
+      ::new (&locale_) std::locale(loc.template get<std::locale>());
+  }
+  ~get_locale() {
+    if (has_locale_) locale_.~locale();
+  }
+  operator const std::locale&() const {
+    return has_locale_ ? locale_ : get_classic_locale();
+  }
+};
+
 template <typename FormatContext, typename OutputIt, typename Rep,
           typename Period>
 struct chrono_formatter {
@@ -944,9 +1585,10 @@ struct chrono_formatter {
   bool negative;
 
   using char_type = typename FormatContext::char_type;
+  using tm_writer_type = tm_writer<OutputIt, char_type>;
 
-  explicit chrono_formatter(FormatContext& ctx, OutputIt o,
-                            std::chrono::duration<Rep, Period> d)
+  chrono_formatter(FormatContext& ctx, OutputIt o,
+                   std::chrono::duration<Rep, Period> d)
       : context(ctx),
         out(o),
         val(static_cast<rep>(d.count())),
@@ -1021,15 +1663,48 @@ struct chrono_formatter {
     out = format_decimal<char_type>(out, n, num_digits).end;
   }
 
+  template <class Duration> void write_fractional_seconds(Duration d) {
+    constexpr auto num_fractional_digits =
+        count_fractional_digits(Duration::period::num, Duration::period::den);
+
+    using subsecond_precision = std::chrono::duration<
+        typename std::common_type<typename Duration::rep,
+                                  std::chrono::seconds::rep>::type,
+        std::ratio<1, detail::pow10(num_fractional_digits)>>;
+    if (std::ratio_less<typename subsecond_precision::period,
+                        std::chrono::seconds::period>::value) {
+      *out++ = '.';
+      // Don't convert long double to integer seconds to avoid overflow.
+      using sec = conditional_t<
+          std::is_same<typename Duration::rep, long double>::value,
+          std::chrono::duration<long double>, std::chrono::seconds>;
+      auto fractional = detail::abs(d) - std::chrono::duration_cast<sec>(d);
+      const auto subseconds =
+          std::chrono::treat_as_floating_point<
+              typename subsecond_precision::rep>::value
+              ? fractional.count()
+              : std::chrono::duration_cast<subsecond_precision>(fractional)
+                    .count();
+      uint32_or_64_or_128_t<long long> n =
+          to_unsigned(to_nonnegative_int(subseconds, max_value<long long>()));
+      int num_digits = detail::count_digits(n);
+      if (num_fractional_digits > num_digits)
+        out = std::fill_n(out, num_fractional_digits - num_digits, '0');
+      out = format_decimal<char_type>(out, n, num_digits).end;
+    }
+  }
+
   void write_nan() { std::copy_n("nan", 3, out); }
   void write_pinf() { std::copy_n("inf", 3, out); }
   void write_ninf() { std::copy_n("-inf", 4, out); }
 
-  void format_localized(const tm& time, char format, char modifier = 0) {
+  template <typename Callback, typename... Args>
+  void format_tm(const tm& time, Callback cb, Args... args) {
     if (isnan(val)) return write_nan();
-    const auto& loc = localized ? context.locale().template get<std::locale>()
-                                : std::locale::classic();
-    out = detail::write(out, time, loc, format, modifier);
+    get_locale loc(localized, context.locale());
+    auto w = tm_writer_type(loc, out, time);
+    (w.*cb)(args...);
+    out = w.out();
   }
 
   void on_text(const char_type* begin, const char_type* end) {
@@ -1050,6 +1725,19 @@ struct chrono_formatter {
   void on_iso_date() {}
   void on_utc_offset() {}
   void on_tz_name() {}
+  void on_year(numeric_system) {}
+  void on_short_year(numeric_system) {}
+  void on_offset_year() {}
+  void on_century(numeric_system) {}
+  void on_iso_week_based_year() {}
+  void on_iso_week_based_short_year() {}
+  void on_dec_month(numeric_system) {}
+  void on_dec0_week_of_year(numeric_system) {}
+  void on_dec1_week_of_year(numeric_system) {}
+  void on_iso_week_of_year(numeric_system) {}
+  void on_day_of_year() {}
+  void on_day_of_month(numeric_system) {}
+  void on_day_of_month_space(numeric_system) {}
 
   void on_24_hour(numeric_system ns) {
     if (handle_nan_inf()) return;
@@ -1057,7 +1745,7 @@ struct chrono_formatter {
     if (ns == numeric_system::standard) return write(hour(), 2);
     auto time = tm();
     time.tm_hour = to_nonnegative_int(hour(), 24);
-    format_localized(time, 'H', 'O');
+    format_tm(time, &tm_writer_type::on_24_hour, ns);
   }
 
   void on_12_hour(numeric_system ns) {
@@ -1066,7 +1754,7 @@ struct chrono_formatter {
     if (ns == numeric_system::standard) return write(hour12(), 2);
     auto time = tm();
     time.tm_hour = to_nonnegative_int(hour12(), 12);
-    format_localized(time, 'I', 'O');
+    format_tm(time, &tm_writer_type::on_12_hour, ns);
   }
 
   void on_minute(numeric_system ns) {
@@ -1075,7 +1763,7 @@ struct chrono_formatter {
     if (ns == numeric_system::standard) return write(minute(), 2);
     auto time = tm();
     time.tm_min = to_nonnegative_int(minute(), 60);
-    format_localized(time, 'M', 'O');
+    format_tm(time, &tm_writer_type::on_minute, ns);
   }
 
   void on_second(numeric_system ns) {
@@ -1083,29 +1771,17 @@ struct chrono_formatter {
 
     if (ns == numeric_system::standard) {
       write(second(), 2);
-#if FMT_SAFE_DURATION_CAST
-      // convert rep->Rep
-      using duration_rep = std::chrono::duration<rep, Period>;
-      using duration_Rep = std::chrono::duration<Rep, Period>;
-      auto tmpval = fmt_safe_duration_cast<duration_Rep>(duration_rep{val});
-#else
-      auto tmpval = std::chrono::duration<Rep, Period>(val);
-#endif
-      auto ms = get_milliseconds(tmpval);
-      if (ms != std::chrono::milliseconds(0)) {
-        *out++ = '.';
-        write(ms.count(), 3);
-      }
+      write_fractional_seconds(std::chrono::duration<rep, Period>{val});
       return;
     }
     auto time = tm();
     time.tm_sec = to_nonnegative_int(second(), 60);
-    format_localized(time, 'S', 'O');
+    format_tm(time, &tm_writer_type::on_second, ns);
   }
 
   void on_12_hour_time() {
     if (handle_nan_inf()) return;
-    format_localized(time(), 'r');
+    format_tm(time(), &tm_writer_type::on_12_hour_time);
   }
 
   void on_24_hour_time() {
@@ -1124,12 +1800,12 @@ struct chrono_formatter {
     on_24_hour_time();
     *out++ = ':';
     if (handle_nan_inf()) return;
-    write(second(), 2);
+    on_second(numeric_system::standard);
   }
 
   void on_am_pm() {
     if (handle_nan_inf()) return;
-    format_localized(time(), 'p');
+    format_tm(time(), &tm_writer_type::on_am_pm);
   }
 
   void on_duration_value() {
@@ -1159,15 +1835,18 @@ class weekday {
       : value(static_cast<unsigned char>(wd != 7 ? wd : 0)) {}
   constexpr unsigned c_encoding() const noexcept { return value; }
 };
+
+class year_month_day {};
 #endif
 
 // A rudimentary weekday formatter.
-template <> struct formatter<weekday> {
+template <typename Char> struct formatter<weekday, Char> {
  private:
   bool localized = false;
 
  public:
-  FMT_CONSTEXPR auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
     auto begin = ctx.begin(), end = ctx.end();
     if (begin != end && *begin == 'L') {
       ++begin;
@@ -1176,12 +1855,14 @@ template <> struct formatter<weekday> {
     return begin;
   }
 
-  auto format(weekday wd, format_context& ctx) -> decltype(ctx.out()) {
+  template <typename FormatContext>
+  auto format(weekday wd, FormatContext& ctx) const -> decltype(ctx.out()) {
     auto time = std::tm();
     time.tm_wday = static_cast<int>(wd.c_encoding());
-    const auto& loc = localized ? ctx.locale().template get<std::locale>()
-                                : std::locale::classic();
-    return detail::write(ctx.out(), time, loc, 'a');
+    detail::get_locale loc(localized, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_abbr_weekday();
+    return w.out();
   }
 };
 
@@ -1260,7 +1941,8 @@ struct formatter<std::chrono::duration<Rep, Period>, Char> {
       ++begin;
       localized = true;
     }
-    end = parse_chrono_format(begin, end, detail::chrono_format_checker());
+    end = detail::parse_chrono_format(begin, end,
+                                      detail::chrono_format_checker());
     return {begin, end};
   }
 
@@ -1302,6 +1984,83 @@ struct formatter<std::chrono::duration<Rep, Period>, Char> {
   }
 };
 
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+                 Char> : formatter<std::tm, Char> {
+  FMT_CONSTEXPR formatter() {
+    this->do_parse(default_specs,
+                   default_specs + sizeof(default_specs) / sizeof(Char));
+  }
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return this->do_parse(ctx.begin(), ctx.end(), true);
+  }
+
+  template <typename FormatContext>
+  auto format(std::chrono::time_point<std::chrono::system_clock> val,
+              FormatContext& ctx) const -> decltype(ctx.out()) {
+    return formatter<std::tm, Char>::format(localtime(val), ctx);
+  }
+
+  static constexpr const Char default_specs[] = {'%', 'F', ' ', '%', 'T'};
+};
+
+template <typename Char, typename Duration>
+constexpr const Char
+    formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+              Char>::default_specs[];
+
+template <typename Char> struct formatter<std::tm, Char> {
+ private:
+  enum class spec {
+    unknown,
+    year_month_day,
+    hh_mm_ss,
+  };
+  spec spec_ = spec::unknown;
+  basic_string_view<Char> specs;
+
+ protected:
+  template <typename It>
+  FMT_CONSTEXPR auto do_parse(It begin, It end, bool with_default = false)
+      -> It {
+    if (begin != end && *begin == ':') ++begin;
+    end = detail::parse_chrono_format(begin, end, detail::tm_format_checker());
+    if (!with_default || end != begin)
+      specs = {begin, detail::to_unsigned(end - begin)};
+    // basic_string_view<>::compare isn't constexpr before C++17.
+    if (specs.size() == 2 && specs[0] == Char('%')) {
+      if (specs[1] == Char('F'))
+        spec_ = spec::year_month_day;
+      else if (specs[1] == Char('T'))
+        spec_ = spec::hh_mm_ss;
+    }
+    return end;
+  }
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return this->do_parse(ctx.begin(), ctx.end());
+  }
+
+  template <typename FormatContext>
+  auto format(const std::tm& tm, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    const auto loc_ref = ctx.locale();
+    detail::get_locale loc(static_cast<bool>(loc_ref), loc_ref);
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), tm);
+    if (spec_ == spec::year_month_day)
+      w.on_iso_date();
+    else if (spec_ == spec::hh_mm_ss)
+      w.on_iso_time();
+    else
+      detail::parse_chrono_format(specs.begin(), specs.end(), w);
+    return w.out();
+  }
+};
+
 FMT_MODULE_EXPORT_END
 FMT_END_NAMESPACE
 
diff --git a/contrib/fmt-8.0.1/include/fmt/color.h b/contrib/fmt-8.1.1/include/fmt/color.h
similarity index 95%
rename from contrib/fmt-8.0.1/include/fmt/color.h
rename to contrib/fmt-8.1.1/include/fmt/color.h
index 3d5490e87f..dfbe482938 100644
--- a/contrib/fmt-8.0.1/include/fmt/color.h
+++ b/contrib/fmt-8.1.1/include/fmt/color.h
@@ -185,9 +185,13 @@ enum class terminal_color : uint8_t {
 
 enum class emphasis : uint8_t {
   bold = 1,
-  italic = 1 << 1,
-  underline = 1 << 2,
-  strikethrough = 1 << 3
+  faint = 1 << 1,
+  italic = 1 << 2,
+  underline = 1 << 3,
+  blink = 1 << 4,
+  reverse = 1 << 5,
+  conceal = 1 << 6,
+  strikethrough = 1 << 7,
 };
 
 // rgb is a struct for red, green and blue colors.
@@ -409,16 +413,18 @@ template <typename Char> struct ansi_color_escape {
     buffer[19] = static_cast<Char>(0);
   }
   FMT_CONSTEXPR ansi_color_escape(emphasis em) FMT_NOEXCEPT {
-    uint8_t em_codes[4] = {};
-    uint8_t em_bits = static_cast<uint8_t>(em);
-    if (em_bits & static_cast<uint8_t>(emphasis::bold)) em_codes[0] = 1;
-    if (em_bits & static_cast<uint8_t>(emphasis::italic)) em_codes[1] = 3;
-    if (em_bits & static_cast<uint8_t>(emphasis::underline)) em_codes[2] = 4;
-    if (em_bits & static_cast<uint8_t>(emphasis::strikethrough))
-      em_codes[3] = 9;
+    uint8_t em_codes[num_emphases] = {};
+    if (has_emphasis(em, emphasis::bold)) em_codes[0] = 1;
+    if (has_emphasis(em, emphasis::faint)) em_codes[1] = 2;
+    if (has_emphasis(em, emphasis::italic)) em_codes[2] = 3;
+    if (has_emphasis(em, emphasis::underline)) em_codes[3] = 4;
+    if (has_emphasis(em, emphasis::blink)) em_codes[4] = 5;
+    if (has_emphasis(em, emphasis::reverse)) em_codes[5] = 7;
+    if (has_emphasis(em, emphasis::conceal)) em_codes[6] = 8;
+    if (has_emphasis(em, emphasis::strikethrough)) em_codes[7] = 9;
 
     size_t index = 0;
-    for (int i = 0; i < 4; ++i) {
+    for (size_t i = 0; i < num_emphases; ++i) {
       if (!em_codes[i]) continue;
       buffer[index++] = static_cast<Char>('\x1b');
       buffer[index++] = static_cast<Char>('[');
@@ -435,7 +441,8 @@ template <typename Char> struct ansi_color_escape {
   }
 
  private:
-  Char buffer[7u + 3u * 4u + 1u];
+  static constexpr size_t num_emphases = 8;
+  Char buffer[7u + 3u * num_emphases + 1u];
 
   static FMT_CONSTEXPR void to_esc(uint8_t c, Char* out,
                                    char delimiter) FMT_NOEXCEPT {
@@ -444,6 +451,10 @@ template <typename Char> struct ansi_color_escape {
     out[2] = static_cast<Char>('0' + c % 10);
     out[3] = static_cast<Char>(delimiter);
   }
+  static FMT_CONSTEXPR bool has_emphasis(emphasis em,
+                                         emphasis mask) FMT_NOEXCEPT {
+    return static_cast<uint8_t>(em) & static_cast<uint8_t>(mask);
+  }
 };
 
 template <typename Char>
diff --git a/contrib/fmt-8.0.1/include/fmt/compile.h b/contrib/fmt-8.1.1/include/fmt/compile.h
similarity index 96%
rename from contrib/fmt-8.0.1/include/fmt/compile.h
rename to contrib/fmt-8.1.1/include/fmt/compile.h
index 00000c92e3..1dba3ddb52 100644
--- a/contrib/fmt-8.0.1/include/fmt/compile.h
+++ b/contrib/fmt-8.1.1/include/fmt/compile.h
@@ -156,7 +156,7 @@ struct is_compiled_string : std::is_base_of<compiled_string, S> {};
     std::string s = fmt::format(FMT_COMPILE("{}"), 42);
   \endrst
  */
-#ifdef __cpp_if_constexpr
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
 #  define FMT_COMPILE(s) \
     FMT_STRING_IMPL(s, fmt::detail::compiled_string, explicit)
 #else
@@ -179,7 +179,7 @@ const T& first(const T& value, const Tail&...) {
   return value;
 }
 
-#ifdef __cpp_if_constexpr
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
 template <typename... Args> struct type_list {};
 
 // Returns a reference to the argument at index N from [first, rest...].
@@ -190,7 +190,7 @@ constexpr const auto& get([[maybe_unused]] const T& first,
   if constexpr (N == 0)
     return first;
   else
-    return get<N - 1>(rest...);
+    return detail::get<N - 1>(rest...);
 }
 
 template <typename Char, typename... Args>
@@ -202,7 +202,8 @@ constexpr int get_arg_index_by_name(basic_string_view<Char> name,
 template <int N, typename> struct get_type_impl;
 
 template <int N, typename... Args> struct get_type_impl<N, type_list<Args...>> {
-  using type = remove_cvref_t<decltype(get<N>(std::declval<Args>()...))>;
+  using type =
+      remove_cvref_t<decltype(detail::get<N>(std::declval<Args>()...))>;
 };
 
 template <int N, typename T>
@@ -242,7 +243,7 @@ template <typename Char> struct code_unit {
 // This ensures that the argument type is convertible to `const T&`.
 template <typename T, int N, typename... Args>
 constexpr const T& get_arg_checked(const Args&... args) {
-  const auto& arg = get<N>(args...);
+  const auto& arg = detail::get<N>(args...);
   if constexpr (detail::is_named_arg<remove_cvref_t<decltype(arg)>>()) {
     return arg.value;
   } else {
@@ -289,7 +290,7 @@ template <typename Char> struct runtime_named_field {
   constexpr OutputIt format(OutputIt out, const Args&... args) const {
     bool found = (try_format_argument(out, name, args) || ...);
     if (!found) {
-      throw format_error("argument with specified name is not found");
+      FMT_THROW(format_error("argument with specified name is not found"));
     }
     return out;
   }
@@ -399,7 +400,9 @@ template <typename Char> struct arg_id_handler {
     return 0;
   }
 
-  constexpr void on_error(const char* message) { throw format_error(message); }
+  constexpr void on_error(const char* message) {
+    FMT_THROW(format_error(message));
+  }
 };
 
 template <typename Char> struct parse_arg_id_result {
@@ -451,7 +454,7 @@ constexpr auto compile_format_string(S format_str) {
   constexpr auto str = basic_string_view<char_type>(format_str);
   if constexpr (str[POS] == '{') {
     if constexpr (POS + 1 == str.size())
-      throw format_error("unmatched '{' in format string");
+      FMT_THROW(format_error("unmatched '{' in format string"));
     if constexpr (str[POS + 1] == '{') {
       return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
     } else if constexpr (str[POS + 1] == '}' || str[POS + 1] == ':') {
@@ -500,7 +503,7 @@ constexpr auto compile_format_string(S format_str) {
     }
   } else if constexpr (str[POS] == '}') {
     if constexpr (POS + 1 == str.size())
-      throw format_error("unmatched '}' in format string");
+      FMT_THROW(format_error("unmatched '}' in format string"));
     return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
   } else {
     constexpr auto end = parse_text(str, POS + 1);
@@ -527,12 +530,12 @@ constexpr auto compile(S format_str) {
     return result;
   }
 }
-#endif  // __cpp_if_constexpr
+#endif  // defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
 }  // namespace detail
 
 FMT_MODULE_EXPORT_BEGIN
 
-#ifdef __cpp_if_constexpr
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
 
 template <typename CompiledFormat, typename... Args,
           typename Char = typename CompiledFormat::char_type,
diff --git a/contrib/fmt-8.0.1/include/fmt/core.h b/contrib/fmt-8.1.1/include/fmt/core.h
similarity index 82%
rename from contrib/fmt-8.0.1/include/fmt/core.h
rename to contrib/fmt-8.1.1/include/fmt/core.h
index d058398ac9..92a7aa1df6 100644
--- a/contrib/fmt-8.0.1/include/fmt/core.h
+++ b/contrib/fmt-8.1.1/include/fmt/core.h
@@ -8,7 +8,8 @@
 #ifndef FMT_CORE_H_
 #define FMT_CORE_H_
 
-#include <cstdio>  // std::FILE
+#include <cstddef>  // std::byte
+#include <cstdio>   // std::FILE
 #include <cstring>
 #include <iterator>
 #include <limits>
@@ -16,29 +17,33 @@
 #include <type_traits>
 
 // The fmt library version in the form major * 10000 + minor * 100 + patch.
-#define FMT_VERSION 80001
+#define FMT_VERSION 80101
 
-#ifdef __clang__
+#if defined(__clang__) && !defined(__ibmxl__)
 #  define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__)
 #else
 #  define FMT_CLANG_VERSION 0
 #endif
 
-#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && \
+    !defined(__NVCOMPILER)
 #  define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  define FMT_GCC_PRAGMA(arg) _Pragma(arg)
 #else
 #  define FMT_GCC_VERSION 0
-#  define FMT_GCC_PRAGMA(arg)
 #endif
 
-#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__)
-#  define FMT_HAS_GXX_CXX11 FMT_GCC_VERSION
-#else
-#  define FMT_HAS_GXX_CXX11 0
+#ifndef FMT_GCC_PRAGMA
+// Workaround _Pragma bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59884.
+#  if FMT_GCC_VERSION >= 504
+#    define FMT_GCC_PRAGMA(arg) _Pragma(arg)
+#  else
+#    define FMT_GCC_PRAGMA(arg)
+#  endif
 #endif
 
-#if defined(__INTEL_COMPILER)
+#ifdef __ICL
+#  define FMT_ICC_VERSION __ICL
+#elif defined(__INTEL_COMPILER)
 #  define FMT_ICC_VERSION __INTEL_COMPILER
 #else
 #  define FMT_ICC_VERSION 0
@@ -78,17 +83,23 @@
 #  define FMT_HAS_CPP_ATTRIBUTE(x) 0
 #endif
 
+#ifdef _MSVC_LANG
+#  define FMT_CPLUSPLUS _MSVC_LANG
+#else
+#  define FMT_CPLUSPLUS __cplusplus
+#endif
+
 #define FMT_HAS_CPP14_ATTRIBUTE(attribute) \
-  (__cplusplus >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+  (FMT_CPLUSPLUS >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute))
 
 #define FMT_HAS_CPP17_ATTRIBUTE(attribute) \
-  (__cplusplus >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+  (FMT_CPLUSPLUS >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute))
 
 // Check if relaxed C++14 constexpr is supported.
 // GCC doesn't allow throw in constexpr until version 6 (bug 67371).
 #ifndef FMT_USE_CONSTEXPR
 #  define FMT_USE_CONSTEXPR                                           \
-    (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VER >= 1910 || \
+    (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VER >= 1912 || \
      (FMT_GCC_VERSION >= 600 && __cplusplus >= 201402L)) &&           \
         !FMT_NVCC && !FMT_ICC_VERSION
 #endif
@@ -100,6 +111,14 @@
 #  define FMT_CONSTEXPR_DECL
 #endif
 
+#if ((__cplusplus >= 202002L) &&                              \
+     (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE > 9)) || \
+    (__cplusplus >= 201709L && FMT_GCC_VERSION >= 1002)
+#  define FMT_CONSTEXPR20 constexpr
+#else
+#  define FMT_CONSTEXPR20
+#endif
+
 // Check if constexpr std::char_traits<>::compare,length is supported.
 #if defined(__GLIBCXX__)
 #  if __cplusplus >= 201703L && defined(_GLIBCXX_RELEASE) && \
@@ -116,15 +135,6 @@
 #  define FMT_CONSTEXPR_CHAR_TRAITS
 #endif
 
-#ifndef FMT_OVERRIDE
-#  if FMT_HAS_FEATURE(cxx_override_control) || \
-      (FMT_GCC_VERSION >= 408 && FMT_HAS_GXX_CXX11) || FMT_MSC_VER >= 1900
-#    define FMT_OVERRIDE override
-#  else
-#    define FMT_OVERRIDE
-#  endif
-#endif
-
 // Check if exceptions are disabled.
 #ifndef FMT_EXCEPTIONS
 #  if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \
@@ -141,7 +151,7 @@
 #endif
 
 #if FMT_USE_NOEXCEPT || FMT_HAS_FEATURE(cxx_noexcept) || \
-    (FMT_GCC_VERSION >= 408 && FMT_HAS_GXX_CXX11) || FMT_MSC_VER >= 1900
+    FMT_GCC_VERSION >= 408 || FMT_MSC_VER >= 1900
 #  define FMT_DETECTED_NOEXCEPT noexcept
 #  define FMT_HAS_CXX11_NOEXCEPT 1
 #else
@@ -166,14 +176,6 @@
 #  define FMT_NORETURN
 #endif
 
-#ifndef FMT_MAYBE_UNUSED
-#  if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused)
-#    define FMT_MAYBE_UNUSED [[maybe_unused]]
-#  else
-#    define FMT_MAYBE_UNUSED
-#  endif
-#endif
-
 #if __cplusplus == 201103L || __cplusplus == 201402L
 #  if defined(__INTEL_COMPILER) || defined(__PGI)
 #    define FMT_FALLTHROUGH
@@ -185,13 +187,20 @@
 #  else
 #    define FMT_FALLTHROUGH
 #  endif
-#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough) || \
-    (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)
+#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough)
 #  define FMT_FALLTHROUGH [[fallthrough]]
 #else
 #  define FMT_FALLTHROUGH
 #endif
 
+#ifndef FMT_NODISCARD
+#  if FMT_HAS_CPP17_ATTRIBUTE(nodiscard)
+#    define FMT_NODISCARD [[nodiscard]]
+#  else
+#    define FMT_NODISCARD
+#  endif
+#endif
+
 #ifndef FMT_USE_FLOAT
 #  define FMT_USE_FLOAT 1
 #endif
@@ -210,31 +219,27 @@
 #  endif
 #endif
 
-#ifndef FMT_USE_INLINE_NAMESPACES
-#  if FMT_HAS_FEATURE(cxx_inline_namespaces) || FMT_GCC_VERSION >= 404 || \
-      (FMT_MSC_VER >= 1900 && (!defined(_MANAGED) || !_MANAGED))
-#    define FMT_USE_INLINE_NAMESPACES 1
+#ifndef FMT_DEPRECATED
+#  if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900
+#    define FMT_DEPRECATED [[deprecated]]
 #  else
-#    define FMT_USE_INLINE_NAMESPACES 0
+#    if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__)
+#      define FMT_DEPRECATED __attribute__((deprecated))
+#    elif FMT_MSC_VER
+#      define FMT_DEPRECATED __declspec(deprecated)
+#    else
+#      define FMT_DEPRECATED /* deprecated */
+#    endif
 #  endif
 #endif
 
 #ifndef FMT_BEGIN_NAMESPACE
-#  if FMT_USE_INLINE_NAMESPACES
-#    define FMT_INLINE_NAMESPACE inline namespace
-#    define FMT_END_NAMESPACE \
-      }                       \
-      }
-#  else
-#    define FMT_INLINE_NAMESPACE namespace
-#    define FMT_END_NAMESPACE \
-      }                       \
-      using namespace v8;     \
-      }
-#  endif
 #  define FMT_BEGIN_NAMESPACE \
     namespace fmt {           \
-    FMT_INLINE_NAMESPACE v8 {
+    inline namespace v8 {
+#  define FMT_END_NAMESPACE \
+    }                       \
+    }
 #endif
 
 #ifndef FMT_MODULE_EXPORT
@@ -264,12 +269,6 @@
 #  define FMT_API
 #endif
 
-#if FMT_GCC_VERSION
-#  define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
-#else
-#  define FMT_GCC_VISIBILITY_HIDDEN
-#endif
-
 // libc++ supports string_view in pre-c++17.
 #if (FMT_HAS_INCLUDE(<string_view>) &&                       \
      (__cplusplus > 201402L || defined(_LIBCPP_VERSION))) || \
@@ -286,10 +285,11 @@
 #endif
 
 #ifndef FMT_CONSTEVAL
-#  if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \
-       __cplusplus > 201703L) ||                                 \
-      (defined(__cpp_consteval) &&                               \
-       !FMT_MSC_VER)  // consteval is broken in MSVC.
+#  if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) &&      \
+       __cplusplus > 201703L && !defined(__apple_build_version__)) || \
+      (defined(__cpp_consteval) &&                                    \
+       (!FMT_MSC_VER || _MSC_FULL_VER >= 193030704))
+// consteval is broken in MSVC before VS2022 and Apple clang 13.
 #    define FMT_CONSTEVAL consteval
 #    define FMT_HAS_CONSTEVAL
 #  else
@@ -325,6 +325,8 @@ template <bool B> using bool_constant = std::integral_constant<bool, B>;
 template <typename T>
 using remove_reference_t = typename std::remove_reference<T>::type;
 template <typename T>
+using remove_const_t = typename std::remove_const<T>::type;
+template <typename T>
 using remove_cvref_t = typename std::remove_cv<remove_reference_t<T>>::type;
 template <typename T> struct type_identity { using type = T; };
 template <typename T> using type_identity_t = typename type_identity<T>::type;
@@ -333,11 +335,6 @@ struct monostate {
   constexpr monostate() {}
 };
 
-// Suppress "unused variable" warnings with the method described in
-// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/.
-// (void)var does not work on many Intel compilers.
-template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
-
 // An enable_if helper to be used in template parameters which results in much
 // shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed
 // to workaround a bug in MSVC 2019 (see #1140 and #1186).
@@ -349,16 +346,25 @@ template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
 
 FMT_BEGIN_DETAIL_NAMESPACE
 
-constexpr FMT_INLINE auto is_constant_evaluated() FMT_NOEXCEPT -> bool {
+// Suppress "unused variable" warnings with the method described in
+// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/.
+// (void)var does not work on many Intel compilers.
+template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
+
+constexpr FMT_INLINE auto is_constant_evaluated(bool default_value = false)
+    FMT_NOEXCEPT -> bool {
 #ifdef __cpp_lib_is_constant_evaluated
+  ignore_unused(default_value);
   return std::is_constant_evaluated();
 #else
-  return false;
+  return default_value;
 #endif
 }
 
 // A function to suppress "conditional expression is constant" warnings.
-template <typename T> constexpr auto const_check(T value) -> T { return value; }
+template <typename T> constexpr FMT_INLINE auto const_check(T value) -> T {
+  return value;
+}
 
 FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
                                       const char* message);
@@ -367,7 +373,7 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
 #  ifdef NDEBUG
 // FMT_ASSERT is not empty to avoid -Werror=empty-body.
 #    define FMT_ASSERT(condition, message) \
-      ::fmt::ignore_unused((condition), (message))
+      ::fmt::detail::ignore_unused((condition), (message))
 #  else
 #    define FMT_ASSERT(condition, message)                                    \
       ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \
@@ -376,6 +382,12 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
 #  endif
 #endif
 
+#ifdef __cpp_lib_byte
+using byte = std::byte;
+#else
+enum class byte : unsigned char {};
+#endif
+
 #if defined(FMT_USE_STRING_VIEW)
 template <typename Char> using std_string_view = std::basic_string_view<Char>;
 #elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW)
@@ -457,13 +469,12 @@ template <typename Char> class basic_string_view {
    */
   FMT_CONSTEXPR_CHAR_TRAITS
   FMT_INLINE
-  basic_string_view(const Char* s) : data_(s) {
-    if (detail::const_check(std::is_same<Char, char>::value &&
-                            !detail::is_constant_evaluated()))
-      size_ = std::strlen(reinterpret_cast<const char*>(s));
-    else
-      size_ = std::char_traits<Char>::length(s);
-  }
+  basic_string_view(const Char* s)
+      : data_(s),
+        size_(detail::const_check(std::is_same<Char, char>::value &&
+                                  !detail::is_constant_evaluated(true))
+                  ? std::strlen(reinterpret_cast<const char*>(s))
+                  : std::char_traits<Char>::length(s)) {}
 
   /** Constructs a string reference from a ``std::basic_string`` object. */
   template <typename Traits, typename Alloc>
@@ -478,19 +489,19 @@ template <typename Char> class basic_string_view {
                                                       size_(s.size()) {}
 
   /** Returns a pointer to the string data. */
-  constexpr auto data() const -> const Char* { return data_; }
+  constexpr auto data() const FMT_NOEXCEPT -> const Char* { return data_; }
 
   /** Returns the string size. */
-  constexpr auto size() const -> size_t { return size_; }
+  constexpr auto size() const FMT_NOEXCEPT -> size_t { return size_; }
 
-  constexpr auto begin() const -> iterator { return data_; }
-  constexpr auto end() const -> iterator { return data_ + size_; }
+  constexpr auto begin() const FMT_NOEXCEPT -> iterator { return data_; }
+  constexpr auto end() const FMT_NOEXCEPT -> iterator { return data_ + size_; }
 
-  constexpr auto operator[](size_t pos) const -> const Char& {
+  constexpr auto operator[](size_t pos) const FMT_NOEXCEPT -> const Char& {
     return data_[pos];
   }
 
-  FMT_CONSTEXPR void remove_prefix(size_t n) {
+  FMT_CONSTEXPR void remove_prefix(size_t n) FMT_NOEXCEPT {
     data_ += n;
     size_ -= n;
   }
@@ -570,7 +581,7 @@ constexpr auto to_string_view(const S& s)
 FMT_BEGIN_DETAIL_NAMESPACE
 
 void to_string_view(...);
-using fmt::v8::to_string_view;
+using fmt::to_string_view;
 
 // Specifies whether S is a string type convertible to fmt::basic_string_view.
 // It should be a constexpr function but MSVC 2017 fails to compile it in
@@ -597,6 +608,8 @@ FMT_INLINE void check_format_string(const S&) {
 template <typename..., typename S, FMT_ENABLE_IF(is_compile_string<S>::value)>
 void check_format_string(S);
 
+FMT_NORETURN FMT_API void throw_format_error(const char* message);
+
 struct error_handler {
   constexpr error_handler() = default;
   constexpr error_handler(const error_handler&) = default;
@@ -711,6 +724,22 @@ class appender;
 
 FMT_BEGIN_DETAIL_NAMESPACE
 
+template <typename Context, typename T>
+constexpr auto has_const_formatter_impl(T*)
+    -> decltype(typename Context::template formatter_type<T>().format(
+                    std::declval<const T&>(), std::declval<Context&>()),
+                true) {
+  return true;
+}
+template <typename Context>
+constexpr auto has_const_formatter_impl(...) -> bool {
+  return false;
+}
+template <typename T, typename Context>
+constexpr auto has_const_formatter() -> bool {
+  return has_const_formatter_impl<Context>(static_cast<T*>(nullptr));
+}
+
 // Extracts a reference to the container from back_insert_iterator.
 template <typename Container>
 inline auto get_container(std::back_insert_iterator<Container> it)
@@ -730,13 +759,13 @@ FMT_CONSTEXPR auto copy_str(InputIt begin, InputIt end, OutputIt out)
   return out;
 }
 
-template <typename Char, FMT_ENABLE_IF(std::is_same<Char, char>::value)>
-FMT_CONSTEXPR auto copy_str(const Char* begin, const Char* end, Char* out)
-    -> Char* {
-  if (is_constant_evaluated())
-    return copy_str<Char, const Char*, Char*>(begin, end, out);
+template <typename Char, typename T, typename U,
+          FMT_ENABLE_IF(
+              std::is_same<remove_const_t<T>, U>::value&& is_char<U>::value)>
+FMT_CONSTEXPR auto copy_str(T* begin, T* end, U* out) -> U* {
+  if (is_constant_evaluated()) return copy_str<Char, T*, U*>(begin, end, out);
   auto size = to_unsigned(end - begin);
-  memcpy(out, begin, size);
+  memcpy(out, begin, size * sizeof(U));
   return out + size;
 }
 
@@ -757,22 +786,22 @@ template <typename T> class buffer {
   FMT_MSC_WARNING(suppress : 26495)
   buffer(size_t sz) FMT_NOEXCEPT : size_(sz), capacity_(sz) {}
 
-  buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) FMT_NOEXCEPT
-      : ptr_(p),
-        size_(sz),
-        capacity_(cap) {}
+  FMT_CONSTEXPR20 buffer(T* p = nullptr, size_t sz = 0,
+                         size_t cap = 0) FMT_NOEXCEPT : ptr_(p),
+                                                        size_(sz),
+                                                        capacity_(cap) {}
 
-  ~buffer() = default;
+  FMT_CONSTEXPR20 ~buffer() = default;
   buffer(buffer&&) = default;
 
   /** Sets the buffer data and capacity. */
-  void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT {
+  FMT_CONSTEXPR void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT {
     ptr_ = buf_data;
     capacity_ = buf_capacity;
   }
 
   /** Increases the buffer capacity to hold at least *capacity* elements. */
-  virtual void grow(size_t capacity) = 0;
+  virtual FMT_CONSTEXPR20 void grow(size_t capacity) = 0;
 
  public:
   using value_type = T;
@@ -788,23 +817,23 @@ template <typename T> class buffer {
   auto end() const FMT_NOEXCEPT -> const T* { return ptr_ + size_; }
 
   /** Returns the size of this buffer. */
-  auto size() const FMT_NOEXCEPT -> size_t { return size_; }
+  constexpr auto size() const FMT_NOEXCEPT -> size_t { return size_; }
 
   /** Returns the capacity of this buffer. */
-  auto capacity() const FMT_NOEXCEPT -> size_t { return capacity_; }
+  constexpr auto capacity() const FMT_NOEXCEPT -> size_t { return capacity_; }
 
   /** Returns a pointer to the buffer data. */
-  auto data() FMT_NOEXCEPT -> T* { return ptr_; }
+  FMT_CONSTEXPR auto data() FMT_NOEXCEPT -> T* { return ptr_; }
 
   /** Returns a pointer to the buffer data. */
-  auto data() const FMT_NOEXCEPT -> const T* { return ptr_; }
+  FMT_CONSTEXPR auto data() const FMT_NOEXCEPT -> const T* { return ptr_; }
 
   /** Clears this buffer. */
   void clear() { size_ = 0; }
 
   // Tries resizing the buffer to contain *count* elements. If T is a POD type
   // the new elements may not be initialized.
-  void try_resize(size_t count) {
+  FMT_CONSTEXPR20 void try_resize(size_t count) {
     try_reserve(count);
     size_ = count <= capacity_ ? count : capacity_;
   }
@@ -813,11 +842,11 @@ template <typename T> class buffer {
   // capacity by a smaller amount than requested but guarantees there is space
   // for at least one additional element either by increasing the capacity or by
   // flushing the buffer if it is full.
-  void try_reserve(size_t new_capacity) {
+  FMT_CONSTEXPR20 void try_reserve(size_t new_capacity) {
     if (new_capacity > capacity_) grow(new_capacity);
   }
 
-  void push_back(const T& value) {
+  FMT_CONSTEXPR20 void push_back(const T& value) {
     try_reserve(size_ + 1);
     ptr_[size_++] = value;
   }
@@ -825,8 +854,11 @@ template <typename T> class buffer {
   /** Appends data to the end of the buffer. */
   template <typename U> void append(const U* begin, const U* end);
 
-  template <typename I> auto operator[](I index) -> T& { return ptr_[index]; }
-  template <typename I> auto operator[](I index) const -> const T& {
+  template <typename I> FMT_CONSTEXPR auto operator[](I index) -> T& {
+    return ptr_[index];
+  }
+  template <typename I>
+  FMT_CONSTEXPR auto operator[](I index) const -> const T& {
     return ptr_[index];
   }
 };
@@ -861,7 +893,7 @@ class iterator_buffer final : public Traits, public buffer<T> {
   T data_[buffer_size];
 
  protected:
-  void grow(size_t) final FMT_OVERRIDE {
+  FMT_CONSTEXPR20 void grow(size_t) override {
     if (this->size() == buffer_size) flush();
   }
 
@@ -885,9 +917,55 @@ class iterator_buffer final : public Traits, public buffer<T> {
   auto count() const -> size_t { return Traits::count() + this->size(); }
 };
 
+template <typename T>
+class iterator_buffer<T*, T, fixed_buffer_traits> final
+    : public fixed_buffer_traits,
+      public buffer<T> {
+ private:
+  T* out_;
+  enum { buffer_size = 256 };
+  T data_[buffer_size];
+
+ protected:
+  FMT_CONSTEXPR20 void grow(size_t) override {
+    if (this->size() == this->capacity()) flush();
+  }
+
+  void flush() {
+    size_t n = this->limit(this->size());
+    if (this->data() == out_) {
+      out_ += n;
+      this->set(data_, buffer_size);
+    }
+    this->clear();
+  }
+
+ public:
+  explicit iterator_buffer(T* out, size_t n = buffer_size)
+      : fixed_buffer_traits(n), buffer<T>(out, 0, n), out_(out) {}
+  iterator_buffer(iterator_buffer&& other)
+      : fixed_buffer_traits(other),
+        buffer<T>(std::move(other)),
+        out_(other.out_) {
+    if (this->data() != out_) {
+      this->set(data_, buffer_size);
+      this->clear();
+    }
+  }
+  ~iterator_buffer() { flush(); }
+
+  auto out() -> T* {
+    flush();
+    return out_;
+  }
+  auto count() const -> size_t {
+    return fixed_buffer_traits::count() + this->size();
+  }
+};
+
 template <typename T> class iterator_buffer<T*, T> final : public buffer<T> {
  protected:
-  void grow(size_t) final FMT_OVERRIDE {}
+  FMT_CONSTEXPR20 void grow(size_t) override {}
 
  public:
   explicit iterator_buffer(T* out, size_t = 0) : buffer<T>(out, 0, ~size_t()) {}
@@ -905,7 +983,7 @@ class iterator_buffer<std::back_insert_iterator<Container>,
   Container& container_;
 
  protected:
-  void grow(size_t capacity) final FMT_OVERRIDE {
+  FMT_CONSTEXPR20 void grow(size_t capacity) override {
     container_.resize(capacity);
     this->set(&container_[0], capacity);
   }
@@ -928,7 +1006,7 @@ template <typename T = char> class counting_buffer final : public buffer<T> {
   size_t count_ = 0;
 
  protected:
-  void grow(size_t) final FMT_OVERRIDE {
+  FMT_CONSTEXPR20 void grow(size_t) override {
     if (this->size() != buffer_size) return;
     count_ += this->size();
     this->clear();
@@ -1045,6 +1123,11 @@ template <typename... Args> constexpr auto count_named_args() -> size_t {
   return count<is_named_arg<Args>::value...>();
 }
 
+template <typename... Args>
+constexpr auto count_statically_named_args() -> size_t {
+  return count<is_statically_named_arg<Args>::value...>();
+}
+
 enum class type {
   none_type,
   // Integer types should go first,
@@ -1100,6 +1183,11 @@ constexpr bool is_arithmetic_type(type t) {
   return t > type::none_type && t <= type::last_numeric_type;
 }
 
+struct unformattable {};
+struct unformattable_char : unformattable {};
+struct unformattable_const : unformattable {};
+struct unformattable_pointer : unformattable {};
+
 template <typename Char> struct string_value {
   const Char* data;
   size_t size;
@@ -1112,8 +1200,8 @@ template <typename Char> struct named_arg_value {
 
 template <typename Context> struct custom_value {
   using parse_context = typename Context::parse_context_type;
-  const void* value;
-  void (*format)(const void* arg, parse_context& parse_ctx, Context& ctx);
+  void* value;
+  void (*format)(void* arg, parse_context& parse_ctx, Context& ctx);
 };
 
 // A formatting argument value.
@@ -1147,8 +1235,8 @@ template <typename Context> class value {
   constexpr FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {}
   FMT_INLINE value(int128_t val) : int128_value(val) {}
   FMT_INLINE value(uint128_t val) : uint128_value(val) {}
-  FMT_INLINE value(float val) : float_value(val) {}
-  FMT_INLINE value(double val) : double_value(val) {}
+  constexpr FMT_INLINE value(float val) : float_value(val) {}
+  constexpr FMT_INLINE value(double val) : double_value(val) {}
   FMT_INLINE value(long double val) : long_double_value(val) {}
   constexpr FMT_INLINE value(bool val) : bool_value(val) {}
   constexpr FMT_INLINE value(char_type val) : char_value(val) {}
@@ -1164,26 +1252,34 @@ template <typename Context> class value {
   FMT_INLINE value(const named_arg_info<char_type>* args, size_t size)
       : named_args{args, size} {}
 
-  template <typename T> FMT_CONSTEXPR FMT_INLINE value(const T& val) {
-    custom.value = &val;
+  template <typename T> FMT_CONSTEXPR FMT_INLINE value(T& val) {
+    using value_type = remove_cvref_t<T>;
+    custom.value = const_cast<value_type*>(&val);
     // Get the formatter type through the context to allow different contexts
     // have different extension points, e.g. `formatter<T>` for `format` and
     // `printf_formatter<T>` for `printf`.
     custom.format = format_custom_arg<
-        T, conditional_t<has_formatter<T, Context>::value,
-                         typename Context::template formatter_type<T>,
-                         fallback_formatter<T, char_type>>>;
+        value_type,
+        conditional_t<has_formatter<value_type, Context>::value,
+                      typename Context::template formatter_type<value_type>,
+                      fallback_formatter<value_type, char_type>>>;
   }
+  value(unformattable);
+  value(unformattable_char);
+  value(unformattable_const);
+  value(unformattable_pointer);
 
  private:
   // Formats an argument of a custom type, such as a user-defined class.
   template <typename T, typename Formatter>
-  static void format_custom_arg(const void* arg,
+  static void format_custom_arg(void* arg,
                                 typename Context::parse_context_type& parse_ctx,
                                 Context& ctx) {
-    Formatter f;
+    auto f = Formatter();
     parse_ctx.advance_to(f.parse(parse_ctx));
-    ctx.advance_to(f.format(*static_cast<const T*>(arg), ctx));
+    using qualified_type =
+        conditional_t<has_const_formatter<T, Context>(), const T, T>;
+    ctx.advance_to(f.format(*static_cast<qualified_type*>(arg), ctx));
   }
 };
 
@@ -1196,9 +1292,9 @@ enum { long_short = sizeof(long) == sizeof(int) };
 using long_type = conditional_t<long_short, int, long long>;
 using ulong_type = conditional_t<long_short, unsigned, unsigned long long>;
 
-struct unformattable {};
-
 // Maps formatting arguments to core types.
+// arg_mapper reports errors by returning unformattable instead of using
+// static_assert because it's used in the is_formattable trait.
 template <typename Context> struct arg_mapper {
   using char_type = typename Context::char_type;
 
@@ -1225,13 +1321,22 @@ template <typename Context> struct arg_mapper {
   FMT_CONSTEXPR FMT_INLINE auto map(uint128_t val) -> uint128_t { return val; }
   FMT_CONSTEXPR FMT_INLINE auto map(bool val) -> bool { return val; }
 
-  template <typename T, FMT_ENABLE_IF(is_char<T>::value)>
+  template <typename T, FMT_ENABLE_IF(std::is_same<T, char>::value ||
+                                      std::is_same<T, char_type>::value)>
   FMT_CONSTEXPR FMT_INLINE auto map(T val) -> char_type {
-    static_assert(
-        std::is_same<T, char>::value || std::is_same<T, char_type>::value,
-        "mixing character types is disallowed");
     return val;
   }
+  template <typename T, enable_if_t<(std::is_same<T, wchar_t>::value ||
+#ifdef __cpp_char8_t
+                                     std::is_same<T, char8_t>::value ||
+#endif
+                                     std::is_same<T, char16_t>::value ||
+                                     std::is_same<T, char32_t>::value) &&
+                                        !std::is_same<T, char_type>::value,
+                                    int> = 0>
+  FMT_CONSTEXPR FMT_INLINE auto map(T) -> unformattable_char {
+    return {};
+  }
 
   FMT_CONSTEXPR FMT_INLINE auto map(float val) -> float { return val; }
   FMT_CONSTEXPR FMT_INLINE auto map(double val) -> double { return val; }
@@ -1245,13 +1350,19 @@ template <typename Context> struct arg_mapper {
   FMT_CONSTEXPR FMT_INLINE auto map(const char_type* val) -> const char_type* {
     return val;
   }
-  template <typename T, FMT_ENABLE_IF(is_string<T>::value)>
+  template <typename T,
+            FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+                          std::is_same<char_type, char_t<T>>::value)>
   FMT_CONSTEXPR FMT_INLINE auto map(const T& val)
       -> basic_string_view<char_type> {
-    static_assert(std::is_same<char_type, char_t<T>>::value,
-                  "mixing character types is disallowed");
     return to_string_view(val);
   }
+  template <typename T,
+            FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+                          !std::is_same<char_type, char_t<T>>::value)>
+  FMT_CONSTEXPR FMT_INLINE auto map(const T&) -> unformattable_char {
+    return {};
+  }
   template <typename T,
             FMT_ENABLE_IF(
                 std::is_constructible<basic_string_view<char_type>, T>::value &&
@@ -1272,21 +1383,25 @@ template <typename Context> struct arg_mapper {
       -> basic_string_view<char_type> {
     return std_string_view<char_type>(val);
   }
-  FMT_CONSTEXPR FMT_INLINE auto map(const signed char* val) -> const char* {
-    static_assert(std::is_same<char_type, char>::value, "invalid string type");
-    return reinterpret_cast<const char*>(val);
+
+  using cstring_result = conditional_t<std::is_same<char_type, char>::value,
+                                       const char*, unformattable_pointer>;
+
+  FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(const signed char* val)
+      -> cstring_result {
+    return map(reinterpret_cast<const char*>(val));
   }
-  FMT_CONSTEXPR FMT_INLINE auto map(const unsigned char* val) -> const char* {
-    static_assert(std::is_same<char_type, char>::value, "invalid string type");
-    return reinterpret_cast<const char*>(val);
+  FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(const unsigned char* val)
+      -> cstring_result {
+    return map(reinterpret_cast<const char*>(val));
   }
-  FMT_CONSTEXPR FMT_INLINE auto map(signed char* val) -> const char* {
-    const auto* const_val = val;
-    return map(const_val);
+  FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(signed char* val)
+      -> cstring_result {
+    return map(reinterpret_cast<const char*>(val));
   }
-  FMT_CONSTEXPR FMT_INLINE auto map(unsigned char* val) -> const char* {
-    const auto* const_val = val;
-    return map(const_val);
+  FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(unsigned char* val)
+      -> cstring_result {
+    return map(reinterpret_cast<const char*>(val));
   }
 
   FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; }
@@ -1299,37 +1414,69 @@ template <typename Context> struct arg_mapper {
 
   // We use SFINAE instead of a const T* parameter to avoid conflicting with
   // the C array overload.
-  template <typename T>
-  FMT_CONSTEXPR auto map(T) -> enable_if_t<std::is_pointer<T>::value, int> {
-    // Formatting of arbitrary pointers is disallowed. If you want to output
-    // a pointer cast it to "void *" or "const void *". In particular, this
-    // forbids formatting of "[const] volatile char *" which is printed as bool
-    // by iostreams.
-    static_assert(!sizeof(T), "formatting of non-void pointers is disallowed");
-    return 0;
+  template <
+      typename T,
+      FMT_ENABLE_IF(
+          std::is_member_pointer<T>::value ||
+          std::is_function<typename std::remove_pointer<T>::type>::value ||
+          (std::is_convertible<const T&, const void*>::value &&
+           !std::is_convertible<const T&, const char_type*>::value))>
+  FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer {
+    return {};
   }
 
-  template <typename T, std::size_t N>
+  template <typename T, std::size_t N,
+            FMT_ENABLE_IF(!std::is_same<T, wchar_t>::value)>
   FMT_CONSTEXPR FMT_INLINE auto map(const T (&values)[N]) -> const T (&)[N] {
     return values;
   }
 
   template <typename T,
-            FMT_ENABLE_IF(std::is_enum<T>::value &&
-                          !has_formatter<T, Context>::value &&
-                          !has_fallback_formatter<T, char_type>::value)>
+            FMT_ENABLE_IF(
+                std::is_enum<T>::value&& std::is_convertible<T, int>::value &&
+                !has_formatter<T, Context>::value &&
+                !has_fallback_formatter<T, char_type>::value)>
   FMT_CONSTEXPR FMT_INLINE auto map(const T& val)
       -> decltype(std::declval<arg_mapper>().map(
           static_cast<typename std::underlying_type<T>::type>(val))) {
     return map(static_cast<typename std::underlying_type<T>::type>(val));
   }
-  template <typename T,
-            FMT_ENABLE_IF(!is_string<T>::value && !is_char<T>::value &&
-                          (has_formatter<T, Context>::value ||
-                           has_fallback_formatter<T, char_type>::value))>
-  FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> const T& {
+
+  FMT_CONSTEXPR FMT_INLINE auto map(detail::byte val) -> unsigned {
+    return map(static_cast<unsigned char>(val));
+  }
+
+  template <typename T, typename U = remove_cvref_t<T>>
+  struct formattable
+      : bool_constant<has_const_formatter<U, Context>() ||
+                      !std::is_const<remove_reference_t<T>>::value ||
+                      has_fallback_formatter<U, char_type>::value> {};
+
+#if FMT_MSC_VER != 0 && FMT_MSC_VER < 1910
+  // Workaround a bug in MSVC.
+  template <typename T> FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& {
     return val;
   }
+#else
+  template <typename T, FMT_ENABLE_IF(formattable<T>::value)>
+  FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& {
+    return val;
+  }
+  template <typename T, FMT_ENABLE_IF(!formattable<T>::value)>
+  FMT_CONSTEXPR FMT_INLINE auto do_map(T&&) -> unformattable_const {
+    return {};
+  }
+#endif
+
+  template <typename T, typename U = remove_cvref_t<T>,
+            FMT_ENABLE_IF(!is_string<U>::value && !is_char<U>::value &&
+                          !std::is_array<U>::value &&
+                          (has_formatter<U, Context>::value ||
+                           has_fallback_formatter<U, char_type>::value))>
+  FMT_CONSTEXPR FMT_INLINE auto map(T&& val)
+      -> decltype(this->do_map(std::forward<T>(val))) {
+    return do_map(std::forward<T>(val));
+  }
 
   template <typename T, FMT_ENABLE_IF(is_named_arg<T>::value)>
   FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg)
@@ -1366,19 +1513,12 @@ class appender : public std::back_insert_iterator<detail::buffer<char>> {
 
  public:
   using std::back_insert_iterator<detail::buffer<char>>::back_insert_iterator;
-  appender(base it) : base(it) {}
+  appender(base it) FMT_NOEXCEPT : base(it) {}
   using _Unchecked_type = appender;  // Mark iterator as checked.
 
-  auto operator++() -> appender& {
-    base::operator++();
-    return *this;
-  }
+  auto operator++() FMT_NOEXCEPT -> appender& { return *this; }
 
-  auto operator++(int) -> appender {
-    auto tmp = *this;
-    ++*this;
-    return tmp;
-  }
+  auto operator++(int) FMT_NOEXCEPT -> appender { return *this; }
 };
 
 // A formatting argument. It is a trivially copyable/constructible type to
@@ -1562,10 +1702,30 @@ FMT_CONSTEXPR auto make_arg(const T& value) -> basic_format_arg<Context> {
 // another (not recommended).
 template <bool IS_PACKED, typename Context, type, typename T,
           FMT_ENABLE_IF(IS_PACKED)>
-FMT_CONSTEXPR FMT_INLINE auto make_arg(const T& val) -> value<Context> {
-  const auto& arg = arg_mapper<Context>().map(val);
+FMT_CONSTEXPR FMT_INLINE auto make_arg(T&& val) -> value<Context> {
+  const auto& arg = arg_mapper<Context>().map(std::forward<T>(val));
+
+  constexpr bool formattable_char =
+      !std::is_same<decltype(arg), const unformattable_char&>::value;
+  static_assert(formattable_char, "Mixing character types is disallowed.");
+
+  constexpr bool formattable_const =
+      !std::is_same<decltype(arg), const unformattable_const&>::value;
+  static_assert(formattable_const, "Cannot format a const argument.");
+
+  // Formatting of arbitrary pointers is disallowed. If you want to output
+  // a pointer cast it to "void *" or "const void *". In particular, this
+  // forbids formatting of "[const] volatile char *" which is printed as bool
+  // by iostreams.
+  constexpr bool formattable_pointer =
+      !std::is_same<decltype(arg), const unformattable_pointer&>::value;
+  static_assert(formattable_pointer,
+                "Formatting of non-void pointers is disallowed.");
+
+  constexpr bool formattable =
+      !std::is_same<decltype(arg), const unformattable&>::value;
   static_assert(
-      !std::is_same<decltype(arg), const unformattable&>::value,
+      formattable,
       "Cannot format an argument. To make type T formattable provide a "
       "formatter<T> specialization: https://fmt.dev/latest/api.html#udt");
   return {arg};
@@ -1643,9 +1803,9 @@ using format_context = buffer_context<char>;
 
 template <typename T, typename Char = char>
 using is_formattable = bool_constant<
-    !std::is_same<decltype(detail::arg_mapper<buffer_context<Char>>().map(
-                      std::declval<T>())),
-                  detail::unformattable>::value &&
+    !std::is_base_of<detail::unformattable,
+                     decltype(detail::arg_mapper<buffer_context<Char>>().map(
+                         std::declval<T>()))>::value &&
     !detail::has_fallback_formatter<T, Char>::value>;
 
 /**
@@ -1684,14 +1844,16 @@ class format_arg_store
            : 0);
 
  public:
-  FMT_CONSTEXPR FMT_INLINE format_arg_store(const Args&... args)
+  template <typename... T>
+  FMT_CONSTEXPR FMT_INLINE format_arg_store(T&&... args)
       :
 #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
         basic_format_args<Context>(*this),
 #endif
         data_{detail::make_arg<
             is_packed, Context,
-            detail::mapped_type_constant<Args, Context>::value>(args)...} {
+            detail::mapped_type_constant<remove_cvref_t<T>, Context>::value>(
+            std::forward<T>(args))...} {
     detail::init_named_args(data_.named_args(), 0, 0, args...);
   }
 };
@@ -1705,9 +1867,9 @@ class format_arg_store
   \endrst
  */
 template <typename Context = format_context, typename... Args>
-constexpr auto make_format_args(const Args&... args)
-    -> format_arg_store<Context, Args...> {
-  return {args...};
+constexpr auto make_format_args(Args&&... args)
+    -> format_arg_store<Context, remove_cvref_t<Args>...> {
+  return {std::forward<Args>(args)...};
 }
 
 /**
@@ -1866,8 +2028,6 @@ using sign_t = sign::type;
 
 FMT_BEGIN_DETAIL_NAMESPACE
 
-void throw_format_error(const char* message);
-
 // Workaround an array initialization issue in gcc 4.8.
 template <typename Char> struct fill_t {
  private:
@@ -1893,11 +2053,33 @@ template <typename Char> struct fill_t {
 };
 FMT_END_DETAIL_NAMESPACE
 
+enum class presentation_type : unsigned char {
+  none,
+  // Integer types should go first,
+  dec,             // 'd'
+  oct,             // 'o'
+  hex_lower,       // 'x'
+  hex_upper,       // 'X'
+  bin_lower,       // 'b'
+  bin_upper,       // 'B'
+  hexfloat_lower,  // 'a'
+  hexfloat_upper,  // 'A'
+  exp_lower,       // 'e'
+  exp_upper,       // 'E'
+  fixed_lower,     // 'f'
+  fixed_upper,     // 'F'
+  general_lower,   // 'g'
+  general_upper,   // 'G'
+  chr,             // 'c'
+  string,          // 's'
+  pointer          // 'p'
+};
+
 // Format specifiers for built-in and string types.
 template <typename Char> struct basic_format_specs {
   int width;
   int precision;
-  char type;
+  presentation_type type;
   align_t align : 4;
   sign_t sign : 3;
   bool alt : 1;  // Alternate form ('#').
@@ -1907,7 +2089,7 @@ template <typename Char> struct basic_format_specs {
   constexpr basic_format_specs()
       : width(0),
         precision(-1),
-        type(0),
+        type(presentation_type::none),
         align(align::none),
         sign(sign::none),
         alt(false),
@@ -1987,9 +2169,7 @@ template <typename Char> class specs_setter {
   }
   FMT_CONSTEXPR void end_precision() {}
 
-  FMT_CONSTEXPR void on_type(Char type) {
-    specs_.type = static_cast<char>(type);
-  }
+  FMT_CONSTEXPR void on_type(presentation_type type) { specs_.type = type; }
 };
 
 // Format spec handler that saves references to arguments representing dynamic
@@ -2063,8 +2243,8 @@ constexpr auto to_ascii(Char value) ->
 template <typename Char>
 FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int {
   if (const_check(sizeof(Char) != 1)) return 1;
-  constexpr char lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-                              0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0};
+  auto lengths =
+      "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4";
   int len = lengths[static_cast<unsigned char>(*begin) >> 3];
 
   // Compute the pointer to the next character early so that the next
@@ -2271,6 +2451,48 @@ FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end,
   return begin;
 }
 
+template <typename Char>
+FMT_CONSTEXPR auto parse_presentation_type(Char type) -> presentation_type {
+  switch (to_ascii(type)) {
+  case 'd':
+    return presentation_type::dec;
+  case 'o':
+    return presentation_type::oct;
+  case 'x':
+    return presentation_type::hex_lower;
+  case 'X':
+    return presentation_type::hex_upper;
+  case 'b':
+    return presentation_type::bin_lower;
+  case 'B':
+    return presentation_type::bin_upper;
+  case 'a':
+    return presentation_type::hexfloat_lower;
+  case 'A':
+    return presentation_type::hexfloat_upper;
+  case 'e':
+    return presentation_type::exp_lower;
+  case 'E':
+    return presentation_type::exp_upper;
+  case 'f':
+    return presentation_type::fixed_lower;
+  case 'F':
+    return presentation_type::fixed_upper;
+  case 'g':
+    return presentation_type::general_lower;
+  case 'G':
+    return presentation_type::general_upper;
+  case 'c':
+    return presentation_type::chr;
+  case 's':
+    return presentation_type::string;
+  case 'p':
+    return presentation_type::pointer;
+  default:
+    return presentation_type::none;
+  }
+}
+
 // Parses standard format specifiers and sends notifications about parsed
 // components to handler.
 template <typename Char, typename SpecHandler>
@@ -2278,9 +2500,12 @@ FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(const Char* begin,
                                                  const Char* end,
                                                  SpecHandler&& handler)
     -> const Char* {
-  if (begin + 1 < end && begin[1] == '}' && is_ascii_letter(*begin) &&
+  if (1 < end - begin && begin[1] == '}' && is_ascii_letter(*begin) &&
       *begin != 'L') {
-    handler.on_type(*begin++);
+    presentation_type type = parse_presentation_type(*begin++);
+    if (type == presentation_type::none)
+      handler.on_error("invalid type specifier");
+    handler.on_type(type);
     return begin;
   }
 
@@ -2334,7 +2559,12 @@ FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(const Char* begin,
   }
 
   // Parse type.
-  if (begin != end && *begin != '}') handler.on_type(*begin++);
+  if (begin != end && *begin != '}') {
+    presentation_type type = parse_presentation_type(*begin++);
+    if (type == presentation_type::none)
+      handler.on_error("invalid type specifier");
+    handler.on_type(type);
+  }
   return begin;
 }
 
@@ -2381,7 +2611,7 @@ FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end,
 template <bool IS_CONSTEXPR, typename Char, typename Handler>
 FMT_CONSTEXPR FMT_INLINE void parse_format_string(
     basic_string_view<Char> format_str, Handler&& handler) {
-  // this is most likely a name-lookup defect in msvc's modules implementation
+  // Workaround a name-lookup bug in MSVC's modules implementation.
   using detail::find;
 
   auto begin = format_str.data();
@@ -2476,28 +2706,18 @@ class compile_parse_context
 };
 
 template <typename ErrorHandler>
-FMT_CONSTEXPR void check_int_type_spec(char spec, ErrorHandler&& eh) {
-  switch (spec) {
-  case 0:
-  case 'd':
-  case 'x':
-  case 'X':
-  case 'b':
-  case 'B':
-  case 'o':
-  case 'c':
-    break;
-  default:
+FMT_CONSTEXPR void check_int_type_spec(presentation_type type,
+                                       ErrorHandler&& eh) {
+  if (type > presentation_type::bin_upper && type != presentation_type::chr)
     eh.on_error("invalid type specifier");
-    break;
-  }
 }
 
 // Checks char specs and returns true if the type spec is char (and not int).
 template <typename Char, typename ErrorHandler = error_handler>
 FMT_CONSTEXPR auto check_char_specs(const basic_format_specs<Char>& specs,
                                     ErrorHandler&& eh = {}) -> bool {
-  if (specs.type && specs.type != 'c') {
+  if (specs.type != presentation_type::none &&
+      specs.type != presentation_type::chr) {
     check_int_type_spec(specs.type, eh);
     return false;
   }
@@ -2521,7 +2741,7 @@ struct float_specs {
   bool upper : 1;
   bool locale : 1;
   bool binary32 : 1;
-  bool use_grisu : 1;
+  bool fallback : 1;
   bool showpoint : 1;
 };
 
@@ -2533,33 +2753,33 @@ FMT_CONSTEXPR auto parse_float_type_spec(const basic_format_specs<Char>& specs,
   result.showpoint = specs.alt;
   result.locale = specs.localized;
   switch (specs.type) {
-  case 0:
+  case presentation_type::none:
     result.format = float_format::general;
     break;
-  case 'G':
+  case presentation_type::general_upper:
     result.upper = true;
     FMT_FALLTHROUGH;
-  case 'g':
+  case presentation_type::general_lower:
     result.format = float_format::general;
     break;
-  case 'E':
+  case presentation_type::exp_upper:
     result.upper = true;
     FMT_FALLTHROUGH;
-  case 'e':
+  case presentation_type::exp_lower:
     result.format = float_format::exp;
     result.showpoint |= specs.precision != 0;
     break;
-  case 'F':
+  case presentation_type::fixed_upper:
     result.upper = true;
     FMT_FALLTHROUGH;
-  case 'f':
+  case presentation_type::fixed_lower:
     result.format = float_format::fixed;
     result.showpoint |= specs.precision != 0;
     break;
-  case 'A':
+  case presentation_type::hexfloat_upper:
     result.upper = true;
     FMT_FALLTHROUGH;
-  case 'a':
+  case presentation_type::hexfloat_lower:
     result.format = float_format::hex;
     break;
   default:
@@ -2569,22 +2789,27 @@ FMT_CONSTEXPR auto parse_float_type_spec(const basic_format_specs<Char>& specs,
   return result;
 }
 
-template <typename Char, typename ErrorHandler = error_handler>
-FMT_CONSTEXPR auto check_cstring_type_spec(Char spec, ErrorHandler&& eh = {})
-    -> bool {
-  if (spec == 0 || spec == 's') return true;
-  if (spec != 'p') eh.on_error("invalid type specifier");
+template <typename ErrorHandler = error_handler>
+FMT_CONSTEXPR auto check_cstring_type_spec(presentation_type type,
+                                           ErrorHandler&& eh = {}) -> bool {
+  if (type == presentation_type::none || type == presentation_type::string)
+    return true;
+  if (type != presentation_type::pointer) eh.on_error("invalid type specifier");
   return false;
 }
 
-template <typename Char, typename ErrorHandler = error_handler>
-FMT_CONSTEXPR void check_string_type_spec(Char spec, ErrorHandler&& eh = {}) {
-  if (spec != 0 && spec != 's') eh.on_error("invalid type specifier");
+template <typename ErrorHandler = error_handler>
+FMT_CONSTEXPR void check_string_type_spec(presentation_type type,
+                                          ErrorHandler&& eh = {}) {
+  if (type != presentation_type::none && type != presentation_type::string)
+    eh.on_error("invalid type specifier");
 }
 
-template <typename Char, typename ErrorHandler>
-FMT_CONSTEXPR void check_pointer_type_spec(Char spec, ErrorHandler&& eh) {
-  if (spec != 0 && spec != 'p') eh.on_error("invalid type specifier");
+template <typename ErrorHandler>
+FMT_CONSTEXPR void check_pointer_type_spec(presentation_type type,
+                                           ErrorHandler&& eh) {
+  if (type != presentation_type::none && type != presentation_type::pointer)
+    eh.on_error("invalid type specifier");
 }
 
 // A parse_format_specs handler that checks if specifiers are consistent with
@@ -2645,28 +2870,21 @@ constexpr auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
   if constexpr (detail::is_statically_named_arg<T>()) {
     if (name == T::name) return N;
   }
-  if constexpr (sizeof...(Args) > 0) {
+  if constexpr (sizeof...(Args) > 0)
     return get_arg_index_by_name<N + 1, Args...>(name);
-  } else {
-    (void)name;  // Workaround an MSVC bug about "unused" parameter.
-    return invalid_arg_index;
-  }
+  (void)name;  // Workaround an MSVC bug about "unused" parameter.
+  return invalid_arg_index;
 }
 #endif
 
 template <typename... Args, typename Char>
 FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
 #if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS
-  if constexpr (sizeof...(Args) > 0) {
+  if constexpr (sizeof...(Args) > 0)
     return get_arg_index_by_name<0, Args...>(name);
-  } else {
-    (void)name;
-    return invalid_arg_index;
-  }
-#else
+#endif
   (void)name;
   return invalid_arg_index;
-#endif
 }
 
 template <typename Char, typename ErrorHandler, typename... Args>
@@ -2769,7 +2987,10 @@ struct formatter<T, Char,
       FMT_ASSERT(false, "invalid argument type");
       break;
     case detail::type::bool_type:
-      if (!specs_.type || specs_.type == 's') break;
+      if (specs_.type == presentation_type::none ||
+          specs_.type == presentation_type::string) {
+        break;
+      }
       FMT_FALLTHROUGH;
     case detail::type::int_type:
     case detail::type::uint_type:
@@ -2824,6 +3045,7 @@ struct formatter<T, Char,
 
 template <typename Char> struct basic_runtime { basic_string_view<Char> str; };
 
+/** A compile-time format string. */
 template <typename Char, typename... Args> class basic_format_string {
  private:
   basic_string_view<Char> str_;
@@ -2832,14 +3054,15 @@ template <typename Char, typename... Args> class basic_format_string {
   template <typename S,
             FMT_ENABLE_IF(
                 std::is_convertible<const S&, basic_string_view<Char>>::value)>
-  FMT_CONSTEVAL basic_format_string(const S& s) : str_(s) {
+  FMT_CONSTEVAL FMT_INLINE basic_format_string(const S& s) : str_(s) {
     static_assert(
         detail::count<
             (std::is_base_of<detail::view, remove_reference_t<Args>>::value &&
              std::is_reference<Args>::value)...>() == 0,
         "passing views as lvalues is disallowed");
 #ifdef FMT_HAS_CONSTEVAL
-    if constexpr (detail::count_named_args<Args...>() == 0) {
+    if constexpr (detail::count_named_args<Args...>() ==
+                  detail::count_statically_named_args<Args...>()) {
       using checker = detail::format_string_checker<Char, detail::error_handler,
                                                     remove_cvref_t<Args>...>;
       detail::parse_format_string<true>(str_, checker(s, {}));
@@ -2862,7 +3085,16 @@ template <typename S> auto runtime(const S& s) -> basic_string_view<char_t<S>> {
 #else
 template <typename... Args>
 using format_string = basic_format_string<char, type_identity_t<Args>...>;
-// Creates a runtime format string.
+/**
+  \rst
+  Creates a runtime format string.
+
+  **Example**::
+
+    // Check format string at runtime instead of compile-time.
+    fmt::print(fmt::runtime("{:d}"), "I am not a number");
+  \endrst
+ */
 template <typename S> auto runtime(const S& s) -> basic_runtime<char_t<S>> {
   return {{s}};
 }
@@ -2878,11 +3110,12 @@ FMT_API auto vformat(string_view fmt, format_args args) -> std::string;
   **Example**::
 
     #include <fmt/core.h>
-    std::string message = fmt::format("The answer is {}", 42);
+    std::string message = fmt::format("The answer is {}.", 42);
   \endrst
 */
 template <typename... T>
-FMT_INLINE auto format(format_string<T...> fmt, T&&... args) -> std::string {
+FMT_NODISCARD FMT_INLINE auto format(format_string<T...> fmt, T&&... args)
+    -> std::string {
   return vformat(fmt, fmt::make_format_args(args...));
 }
 
@@ -2892,7 +3125,7 @@ template <typename OutputIt,
 auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt {
   using detail::get_buffer;
   auto&& buf = get_buffer<char>(out);
-  detail::vformat_to(buf, string_view(fmt), args, {});
+  detail::vformat_to(buf, fmt, args, {});
   return detail::get_iterator(buf);
 }
 
@@ -2900,7 +3133,7 @@ auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt {
  \rst
  Formats ``args`` according to specifications in ``fmt``, writes the result to
  the output iterator ``out`` and returns the iterator past the end of the output
- range.
+ range. `format_to` does not append a terminating null character.
 
  **Example**::
 
@@ -2926,9 +3159,8 @@ template <typename OutputIt, typename... T,
           FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
 auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args)
     -> format_to_n_result<OutputIt> {
-  using buffer =
-      detail::iterator_buffer<OutputIt, char, detail::fixed_buffer_traits>;
-  auto buf = buffer(out, n);
+  using traits = detail::fixed_buffer_traits;
+  auto buf = detail::iterator_buffer<OutputIt, char, traits>(out, n);
   detail::vformat_to(buf, fmt, args, {});
   return {buf.out(), buf.count()};
 }
@@ -2938,18 +3170,20 @@ auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args)
   Formats ``args`` according to specifications in ``fmt``, writes up to ``n``
   characters of the result to the output iterator ``out`` and returns the total
   (not truncated) output size and the iterator past the end of the output range.
+  `format_to_n` does not append a terminating null character.
   \endrst
  */
 template <typename OutputIt, typename... T,
           FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
 FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string<T...> fmt,
-                            const T&... args) -> format_to_n_result<OutputIt> {
+                            T&&... args) -> format_to_n_result<OutputIt> {
   return vformat_to_n(out, n, fmt, fmt::make_format_args(args...));
 }
 
 /** Returns the number of chars in the output of ``format(fmt, args...)``. */
 template <typename... T>
-FMT_INLINE auto formatted_size(format_string<T...> fmt, T&&... args) -> size_t {
+FMT_NODISCARD FMT_INLINE auto formatted_size(format_string<T...> fmt,
+                                             T&&... args) -> size_t {
   auto buf = detail::counting_buffer<>();
   detail::vformat_to(buf, string_view(fmt), fmt::make_format_args(args...), {});
   return buf.count();
diff --git a/contrib/fmt-8.0.1/include/fmt/format-inl.h b/contrib/fmt-8.1.1/include/fmt/format-inl.h
similarity index 90%
rename from contrib/fmt-8.0.1/include/fmt/format-inl.h
rename to contrib/fmt-8.1.1/include/fmt/format-inl.h
index 94a36d1bc4..2c51c50aeb 100644
--- a/contrib/fmt-8.0.1/include/fmt/format-inl.h
+++ b/contrib/fmt-8.1.1/include/fmt/format-inl.h
@@ -40,6 +40,10 @@ FMT_FUNC void assert_fail(const char* file, int line, const char* message) {
   std::terminate();
 }
 
+FMT_FUNC void throw_format_error(const char* message) {
+  FMT_THROW(format_error(message));
+}
+
 #ifndef _MSC_VER
 #  define FMT_SNPRINTF snprintf
 #else  // _MSC_VER
@@ -145,141 +149,13 @@ template <> FMT_FUNC int count_digits<4>(detail::fallback_uintptr n) {
   return i >= 0 ? i * char_digits + count_digits<4, unsigned>(n.value[i]) : 1;
 }
 
-#if __cplusplus < 201703L
-template <typename T> constexpr const char basic_data<T>::digits[][2];
-template <typename T> constexpr const char basic_data<T>::hex_digits[];
-template <typename T> constexpr const char basic_data<T>::signs[];
-template <typename T> constexpr const unsigned basic_data<T>::prefixes[];
-template <typename T> constexpr const char basic_data<T>::left_padding_shifts[];
-template <typename T>
-constexpr const char basic_data<T>::right_padding_shifts[];
-#endif
+// log10(2) = 0x0.4d104d427de7fbcc...
+static constexpr uint64_t log10_2_significand = 0x4d104d427de7fbcc;
 
-template <typename T> struct bits {
-  static FMT_CONSTEXPR_DECL const int value =
-      static_cast<int>(sizeof(T) * std::numeric_limits<unsigned char>::digits);
-};
-
-class fp;
-template <int SHIFT = 0> fp normalize(fp value);
-
-// Lower (upper) boundary is a value half way between a floating-point value
-// and its predecessor (successor). Boundaries have the same exponent as the
-// value so only significands are stored.
-struct boundaries {
-  uint64_t lower;
-  uint64_t upper;
-};
-
-// A handmade floating-point number f * pow(2, e).
-class fp {
- private:
-  using significand_type = uint64_t;
-
-  template <typename Float>
-  using is_supported_float = bool_constant<sizeof(Float) == sizeof(uint64_t) ||
-                                           sizeof(Float) == sizeof(uint32_t)>;
-
- public:
-  significand_type f;
-  int e;
-
-  // All sizes are in bits.
-  // Subtract 1 to account for an implicit most significant bit in the
-  // normalized form.
-  static FMT_CONSTEXPR_DECL const int double_significand_size =
-      std::numeric_limits<double>::digits - 1;
-  static FMT_CONSTEXPR_DECL const uint64_t implicit_bit =
-      1ULL << double_significand_size;
-  static FMT_CONSTEXPR_DECL const int significand_size =
-      bits<significand_type>::value;
-
-  fp() : f(0), e(0) {}
-  fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {}
-
-  // Constructs fp from an IEEE754 double. It is a template to prevent compile
-  // errors on platforms where double is not IEEE754.
-  template <typename Double> explicit fp(Double d) { assign(d); }
-
-  // Assigns d to this and return true iff predecessor is closer than successor.
-  template <typename Float, FMT_ENABLE_IF(is_supported_float<Float>::value)>
-  bool assign(Float d) {
-    // Assume float is in the format [sign][exponent][significand].
-    using limits = std::numeric_limits<Float>;
-    const int float_significand_size = limits::digits - 1;
-    const int exponent_size =
-        bits<Float>::value - float_significand_size - 1;  // -1 for sign
-    const uint64_t float_implicit_bit = 1ULL << float_significand_size;
-    const uint64_t significand_mask = float_implicit_bit - 1;
-    const uint64_t exponent_mask = (~0ULL >> 1) & ~significand_mask;
-    const int exponent_bias = (1 << exponent_size) - limits::max_exponent - 1;
-    constexpr bool is_double = sizeof(Float) == sizeof(uint64_t);
-    auto u = bit_cast<conditional_t<is_double, uint64_t, uint32_t>>(d);
-    f = u & significand_mask;
-    int biased_e =
-        static_cast<int>((u & exponent_mask) >> float_significand_size);
-    // Predecessor is closer if d is a normalized power of 2 (f == 0) other than
-    // the smallest normalized number (biased_e > 1).
-    bool is_predecessor_closer = f == 0 && biased_e > 1;
-    if (biased_e != 0)
-      f += float_implicit_bit;
-    else
-      biased_e = 1;  // Subnormals use biased exponent 1 (min exponent).
-    e = biased_e - exponent_bias - float_significand_size;
-    return is_predecessor_closer;
-  }
-
-  template <typename Float, FMT_ENABLE_IF(!is_supported_float<Float>::value)>
-  bool assign(Float) {
-    *this = fp();
-    return false;
-  }
-};
-
-// Normalizes the value converted from double and multiplied by (1 << SHIFT).
-template <int SHIFT> fp normalize(fp value) {
-  // Handle subnormals.
-  const auto shifted_implicit_bit = fp::implicit_bit << SHIFT;
-  while ((value.f & shifted_implicit_bit) == 0) {
-    value.f <<= 1;
-    --value.e;
-  }
-  // Subtract 1 to account for hidden bit.
-  const auto offset =
-      fp::significand_size - fp::double_significand_size - SHIFT - 1;
-  value.f <<= offset;
-  value.e -= offset;
-  return value;
-}
-
-inline bool operator==(fp x, fp y) { return x.f == y.f && x.e == y.e; }
-
-// Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking.
-inline uint64_t multiply(uint64_t lhs, uint64_t rhs) {
-#if FMT_USE_INT128
-  auto product = static_cast<__uint128_t>(lhs) * rhs;
-  auto f = static_cast<uint64_t>(product >> 64);
-  return (static_cast<uint64_t>(product) & (1ULL << 63)) != 0 ? f + 1 : f;
-#else
-  // Multiply 32-bit parts of significands.
-  uint64_t mask = (1ULL << 32) - 1;
-  uint64_t a = lhs >> 32, b = lhs & mask;
-  uint64_t c = rhs >> 32, d = rhs & mask;
-  uint64_t ac = a * c, bc = b * c, ad = a * d, bd = b * d;
-  // Compute mid 64-bit of result and round.
-  uint64_t mid = (bd >> 32) + (ad & mask) + (bc & mask) + (1U << 31);
-  return ac + (ad >> 32) + (bc >> 32) + (mid >> 32);
-#endif
-}
-
-inline fp operator*(fp x, fp y) { return {multiply(x.f, y.f), x.e + y.e + 64}; }
-
-// Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its
-// (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`.
-inline fp get_cached_power(int min_exponent, int& pow10_exponent) {
+template <typename T = void> struct basic_impl_data {
   // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340.
   // These are generated by support/compute-powers.py.
-  static constexpr const uint64_t pow10_significands[] = {
+  static constexpr uint64_t pow10_significands[87] = {
       0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76,
       0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df,
       0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c,
@@ -311,9 +187,13 @@ inline fp get_cached_power(int min_exponent, int& pow10_exponent) {
       0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b,
   };
 
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+#  pragma GCC diagnostic push
+#  pragma GCC diagnostic ignored "-Wnarrowing"
+#endif
   // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding
   // to significands above.
-  static constexpr const int16_t pow10_exponents[] = {
+  static constexpr int16_t pow10_exponents[87] = {
       -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954,
       -927,  -901,  -874,  -847,  -821,  -794,  -768,  -741,  -715,  -688, -661,
       -635,  -608,  -582,  -555,  -529,  -502,  -475,  -449,  -422,  -396, -369,
@@ -322,11 +202,137 @@ inline fp get_cached_power(int min_exponent, int& pow10_exponent) {
       242,   269,   295,   322,   348,   375,   402,   428,   455,   481,  508,
       534,   561,   588,   614,   641,   667,   694,   720,   747,   774,  800,
       827,   853,   880,   907,   933,   960,   986,   1013,  1039,  1066};
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+#  pragma GCC diagnostic pop
+#endif
 
+  static constexpr uint64_t power_of_10_64[20] = {
+      1, FMT_POWERS_OF_10(1ULL), FMT_POWERS_OF_10(1000000000ULL),
+      10000000000000000000ULL};
+};
+
+// This is a struct rather than an alias to avoid shadowing warnings in gcc.
+struct impl_data : basic_impl_data<> {};
+
+#if __cplusplus < 201703L
+template <typename T>
+constexpr uint64_t basic_impl_data<T>::pow10_significands[];
+template <typename T> constexpr int16_t basic_impl_data<T>::pow10_exponents[];
+template <typename T> constexpr uint64_t basic_impl_data<T>::power_of_10_64[];
+#endif
+
+template <typename T> struct bits {
+  static FMT_CONSTEXPR_DECL const int value =
+      static_cast<int>(sizeof(T) * std::numeric_limits<unsigned char>::digits);
+};
+
+// Returns the number of significand bits in Float excluding the implicit bit.
+template <typename Float> constexpr int num_significand_bits() {
+  // Subtract 1 to account for an implicit most significant bit in the
+  // normalized form.
+  return std::numeric_limits<Float>::digits - 1;
+}
+
+// A floating-point number f * pow(2, e).
+struct fp {
+  uint64_t f;
+  int e;
+
+  static constexpr const int num_significand_bits = bits<decltype(f)>::value;
+
+  constexpr fp() : f(0), e(0) {}
+  constexpr fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {}
+
+  // Constructs fp from an IEEE754 floating-point number. It is a template to
+  // prevent compile errors on systems where n is not IEEE754.
+  template <typename Float> explicit FMT_CONSTEXPR fp(Float n) { assign(n); }
+
+  template <typename Float>
+  using is_supported = bool_constant<sizeof(Float) == sizeof(uint64_t) ||
+                                     sizeof(Float) == sizeof(uint32_t)>;
+
+  // Assigns d to this and return true iff predecessor is closer than successor.
+  template <typename Float, FMT_ENABLE_IF(is_supported<Float>::value)>
+  FMT_CONSTEXPR bool assign(Float n) {
+    // Assume float is in the format [sign][exponent][significand].
+    const int num_float_significand_bits =
+        detail::num_significand_bits<Float>();
+    const uint64_t implicit_bit = 1ULL << num_float_significand_bits;
+    const uint64_t significand_mask = implicit_bit - 1;
+    constexpr bool is_double = sizeof(Float) == sizeof(uint64_t);
+    auto u = bit_cast<conditional_t<is_double, uint64_t, uint32_t>>(n);
+    f = u & significand_mask;
+    const uint64_t exponent_mask = (~0ULL >> 1) & ~significand_mask;
+    int biased_e =
+        static_cast<int>((u & exponent_mask) >> num_float_significand_bits);
+    // The predecessor is closer if n is a normalized power of 2 (f == 0) other
+    // than the smallest normalized number (biased_e > 1).
+    bool is_predecessor_closer = f == 0 && biased_e > 1;
+    if (biased_e != 0)
+      f += implicit_bit;
+    else
+      biased_e = 1;  // Subnormals use biased exponent 1 (min exponent).
+    const int exponent_bias = std::numeric_limits<Float>::max_exponent - 1;
+    e = biased_e - exponent_bias - num_float_significand_bits;
+    return is_predecessor_closer;
+  }
+
+  template <typename Float, FMT_ENABLE_IF(!is_supported<Float>::value)>
+  bool assign(Float) {
+    FMT_ASSERT(false, "");
+    return false;
+  }
+};
+
+// Normalizes the value converted from double and multiplied by (1 << SHIFT).
+template <int SHIFT = 0> FMT_CONSTEXPR fp normalize(fp value) {
+  // Handle subnormals.
+  const uint64_t implicit_bit = 1ULL << num_significand_bits<double>();
+  const auto shifted_implicit_bit = implicit_bit << SHIFT;
+  while ((value.f & shifted_implicit_bit) == 0) {
+    value.f <<= 1;
+    --value.e;
+  }
+  // Subtract 1 to account for hidden bit.
+  const auto offset =
+      fp::num_significand_bits - num_significand_bits<double>() - SHIFT - 1;
+  value.f <<= offset;
+  value.e -= offset;
+  return value;
+}
+
+inline bool operator==(fp x, fp y) { return x.f == y.f && x.e == y.e; }
+
+// Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking.
+FMT_CONSTEXPR inline uint64_t multiply(uint64_t lhs, uint64_t rhs) {
+#if FMT_USE_INT128
+  auto product = static_cast<__uint128_t>(lhs) * rhs;
+  auto f = static_cast<uint64_t>(product >> 64);
+  return (static_cast<uint64_t>(product) & (1ULL << 63)) != 0 ? f + 1 : f;
+#else
+  // Multiply 32-bit parts of significands.
+  uint64_t mask = (1ULL << 32) - 1;
+  uint64_t a = lhs >> 32, b = lhs & mask;
+  uint64_t c = rhs >> 32, d = rhs & mask;
+  uint64_t ac = a * c, bc = b * c, ad = a * d, bd = b * d;
+  // Compute mid 64-bit of result and round.
+  uint64_t mid = (bd >> 32) + (ad & mask) + (bc & mask) + (1U << 31);
+  return ac + (ad >> 32) + (bc >> 32) + (mid >> 32);
+#endif
+}
+
+FMT_CONSTEXPR inline fp operator*(fp x, fp y) {
+  return {multiply(x.f, y.f), x.e + y.e + 64};
+}
+
+// Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its
+// (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`.
+FMT_CONSTEXPR inline fp get_cached_power(int min_exponent,
+                                         int& pow10_exponent) {
   const int shift = 32;
-  const auto significand = static_cast<int64_t>(data::log10_2_significand);
+  const auto significand = static_cast<int64_t>(log10_2_significand);
   int index = static_cast<int>(
-      ((min_exponent + fp::significand_size - 1) * (significand >> shift) +
+      ((min_exponent + fp::num_significand_bits - 1) * (significand >> shift) +
        ((int64_t(1) << shift) - 1))  // ceil
       >> 32                          // arithmetic shift
   );
@@ -336,7 +342,8 @@ inline fp get_cached_power(int min_exponent, int& pow10_exponent) {
   const int dec_exp_step = 8;
   index = (index - first_dec_exp - 1) / dec_exp_step + 1;
   pow10_exponent = first_dec_exp + index * dec_exp_step;
-  return {pow10_significands[index], pow10_exponents[index]};
+  return {impl_data::pow10_significands[index],
+          impl_data::pow10_exponents[index]};
 }
 
 // A simple accumulator to hold the sums of terms in bigint::square if uint128_t
@@ -345,14 +352,16 @@ struct accumulator {
   uint64_t lower;
   uint64_t upper;
 
-  accumulator() : lower(0), upper(0) {}
-  explicit operator uint32_t() const { return static_cast<uint32_t>(lower); }
+  constexpr accumulator() : lower(0), upper(0) {}
+  constexpr explicit operator uint32_t() const {
+    return static_cast<uint32_t>(lower);
+  }
 
-  void operator+=(uint64_t n) {
+  FMT_CONSTEXPR void operator+=(uint64_t n) {
     lower += n;
     if (lower < n) ++upper;
   }
-  void operator>>=(int shift) {
+  FMT_CONSTEXPR void operator>>=(int shift) {
     FMT_ASSERT(shift == 32, "");
     (void)shift;
     lower = (upper << 32) | (lower >> 32);
@@ -370,27 +379,31 @@ class bigint {
   basic_memory_buffer<bigit, bigits_capacity> bigits_;
   int exp_;
 
-  bigit operator[](int index) const { return bigits_[to_unsigned(index)]; }
-  bigit& operator[](int index) { return bigits_[to_unsigned(index)]; }
+  FMT_CONSTEXPR20 bigit operator[](int index) const {
+    return bigits_[to_unsigned(index)];
+  }
+  FMT_CONSTEXPR20 bigit& operator[](int index) {
+    return bigits_[to_unsigned(index)];
+  }
 
   static FMT_CONSTEXPR_DECL const int bigit_bits = bits<bigit>::value;
 
   friend struct formatter<bigint>;
 
-  void subtract_bigits(int index, bigit other, bigit& borrow) {
+  FMT_CONSTEXPR20 void subtract_bigits(int index, bigit other, bigit& borrow) {
     auto result = static_cast<double_bigit>((*this)[index]) - other - borrow;
     (*this)[index] = static_cast<bigit>(result);
     borrow = static_cast<bigit>(result >> (bigit_bits * 2 - 1));
   }
 
-  void remove_leading_zeros() {
+  FMT_CONSTEXPR20 void remove_leading_zeros() {
     int num_bigits = static_cast<int>(bigits_.size()) - 1;
     while (num_bigits > 0 && (*this)[num_bigits] == 0) --num_bigits;
     bigits_.resize(to_unsigned(num_bigits + 1));
   }
 
   // Computes *this -= other assuming aligned bigints and *this >= other.
-  void subtract_aligned(const bigint& other) {
+  FMT_CONSTEXPR20 void subtract_aligned(const bigint& other) {
     FMT_ASSERT(other.exp_ >= exp_, "unaligned bigints");
     FMT_ASSERT(compare(*this, other) >= 0, "");
     bigit borrow = 0;
@@ -401,7 +414,7 @@ class bigint {
     remove_leading_zeros();
   }
 
-  void multiply(uint32_t value) {
+  FMT_CONSTEXPR20 void multiply(uint32_t value) {
     const double_bigit wide_value = value;
     bigit carry = 0;
     for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
@@ -412,7 +425,7 @@ class bigint {
     if (carry != 0) bigits_.push_back(carry);
   }
 
-  void multiply(uint64_t value) {
+  FMT_CONSTEXPR20 void multiply(uint64_t value) {
     const bigit mask = ~bigit(0);
     const double_bigit lower = value & mask;
     const double_bigit upper = value >> bigit_bits;
@@ -430,14 +443,16 @@ class bigint {
   }
 
  public:
-  bigint() : exp_(0) {}
+  FMT_CONSTEXPR20 bigint() : exp_(0) {}
   explicit bigint(uint64_t n) { assign(n); }
-  ~bigint() { FMT_ASSERT(bigits_.capacity() <= bigits_capacity, ""); }
+  FMT_CONSTEXPR20 ~bigint() {
+    FMT_ASSERT(bigits_.capacity() <= bigits_capacity, "");
+  }
 
   bigint(const bigint&) = delete;
   void operator=(const bigint&) = delete;
 
-  void assign(const bigint& other) {
+  FMT_CONSTEXPR20 void assign(const bigint& other) {
     auto size = other.bigits_.size();
     bigits_.resize(size);
     auto data = other.bigits_.data();
@@ -445,7 +460,7 @@ class bigint {
     exp_ = other.exp_;
   }
 
-  void assign(uint64_t n) {
+  FMT_CONSTEXPR20 void assign(uint64_t n) {
     size_t num_bigits = 0;
     do {
       bigits_[num_bigits++] = n & ~bigit(0);
@@ -455,9 +470,11 @@ class bigint {
     exp_ = 0;
   }
 
-  int num_bigits() const { return static_cast<int>(bigits_.size()) + exp_; }
+  FMT_CONSTEXPR20 int num_bigits() const {
+    return static_cast<int>(bigits_.size()) + exp_;
+  }
 
-  FMT_NOINLINE bigint& operator<<=(int shift) {
+  FMT_NOINLINE FMT_CONSTEXPR20 bigint& operator<<=(int shift) {
     FMT_ASSERT(shift >= 0, "");
     exp_ += shift / bigit_bits;
     shift %= bigit_bits;
@@ -472,13 +489,13 @@ class bigint {
     return *this;
   }
 
-  template <typename Int> bigint& operator*=(Int value) {
+  template <typename Int> FMT_CONSTEXPR20 bigint& operator*=(Int value) {
     FMT_ASSERT(value > 0, "");
     multiply(uint32_or_64_or_128_t<Int>(value));
     return *this;
   }
 
-  friend int compare(const bigint& lhs, const bigint& rhs) {
+  friend FMT_CONSTEXPR20 int compare(const bigint& lhs, const bigint& rhs) {
     int num_lhs_bigits = lhs.num_bigits(), num_rhs_bigits = rhs.num_bigits();
     if (num_lhs_bigits != num_rhs_bigits)
       return num_lhs_bigits > num_rhs_bigits ? 1 : -1;
@@ -495,8 +512,8 @@ class bigint {
   }
 
   // Returns compare(lhs1 + lhs2, rhs).
-  friend int add_compare(const bigint& lhs1, const bigint& lhs2,
-                         const bigint& rhs) {
+  friend FMT_CONSTEXPR20 int add_compare(const bigint& lhs1, const bigint& lhs2,
+                                         const bigint& rhs) {
     int max_lhs_bigits = (std::max)(lhs1.num_bigits(), lhs2.num_bigits());
     int num_rhs_bigits = rhs.num_bigits();
     if (max_lhs_bigits + 1 < num_rhs_bigits) return -1;
@@ -519,7 +536,7 @@ class bigint {
   }
 
   // Assigns pow(10, exp) to this bigint.
-  void assign_pow10(int exp) {
+  FMT_CONSTEXPR20 void assign_pow10(int exp) {
     FMT_ASSERT(exp >= 0, "");
     if (exp == 0) return assign(1);
     // Find the top bit.
@@ -538,7 +555,7 @@ class bigint {
     *this <<= exp;  // Multiply by pow(2, exp) by shifting.
   }
 
-  void square() {
+  FMT_CONSTEXPR20 void square() {
     int num_bigits = static_cast<int>(bigits_.size());
     int num_result_bigits = 2 * num_bigits;
     basic_memory_buffer<bigit, bigits_capacity> n(std::move(bigits_));
@@ -569,7 +586,7 @@ class bigint {
 
   // If this bigint has a bigger exponent than other, adds trailing zero to make
   // exponents equal. This simplifies some operations such as subtraction.
-  void align(const bigint& other) {
+  FMT_CONSTEXPR20 void align(const bigint& other) {
     int exp_difference = exp_ - other.exp_;
     if (exp_difference <= 0) return;
     int num_bigits = static_cast<int>(bigits_.size());
@@ -582,7 +599,7 @@ class bigint {
 
   // Divides this bignum by divisor, assigning the remainder to this and
   // returning the quotient.
-  int divmod_assign(const bigint& divisor) {
+  FMT_CONSTEXPR20 int divmod_assign(const bigint& divisor) {
     FMT_ASSERT(this != &divisor, "");
     if (compare(*this, divisor) < 0) return 0;
     FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, "");
@@ -602,8 +619,9 @@ enum class round_direction { unknown, up, down };
 // some number v and the error, returns whether v should be rounded up, down, or
 // whether the rounding direction can't be determined due to error.
 // error should be less than divisor / 2.
-inline round_direction get_round_direction(uint64_t divisor, uint64_t remainder,
-                                           uint64_t error) {
+FMT_CONSTEXPR inline round_direction get_round_direction(uint64_t divisor,
+                                                         uint64_t remainder,
+                                                         uint64_t error) {
   FMT_ASSERT(remainder < divisor, "");  // divisor - remainder won't overflow.
   FMT_ASSERT(error < divisor, "");      // divisor - error won't overflow.
   FMT_ASSERT(error < divisor - error, "");  // error * 2 won't overflow.
@@ -626,19 +644,52 @@ enum result {
 };
 }
 
-inline uint64_t power_of_10_64(int exp) {
-  static constexpr const uint64_t data[] = {1, FMT_POWERS_OF_10(1),
-                                            FMT_POWERS_OF_10(1000000000ULL),
-                                            10000000000000000000ULL};
-  return data[exp];
-}
+struct gen_digits_handler {
+  char* buf;
+  int size;
+  int precision;
+  int exp10;
+  bool fixed;
+
+  FMT_CONSTEXPR digits::result on_digit(char digit, uint64_t divisor,
+                                        uint64_t remainder, uint64_t error,
+                                        bool integral) {
+    FMT_ASSERT(remainder < divisor, "");
+    buf[size++] = digit;
+    if (!integral && error >= remainder) return digits::error;
+    if (size < precision) return digits::more;
+    if (!integral) {
+      // Check if error * 2 < divisor with overflow prevention.
+      // The check is not needed for the integral part because error = 1
+      // and divisor > (1 << 32) there.
+      if (error >= divisor || error >= divisor - error) return digits::error;
+    } else {
+      FMT_ASSERT(error == 1 && divisor > 2, "");
+    }
+    auto dir = get_round_direction(divisor, remainder, error);
+    if (dir != round_direction::up)
+      return dir == round_direction::down ? digits::done : digits::error;
+    ++buf[size - 1];
+    for (int i = size - 1; i > 0 && buf[i] > '9'; --i) {
+      buf[i] = '0';
+      ++buf[i - 1];
+    }
+    if (buf[0] > '9') {
+      buf[0] = '1';
+      if (fixed)
+        buf[size++] = '0';
+      else
+        ++exp10;
+    }
+    return digits::done;
+  }
+};
 
 // Generates output using the Grisu digit-gen algorithm.
 // error: the size of the region (lower, upper) outside of which numbers
 // definitely do not round to value (Delta in Grisu3).
-template <typename Handler>
-FMT_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, int& exp,
-                                           Handler& handler) {
+FMT_INLINE FMT_CONSTEXPR20 digits::result grisu_gen_digits(
+    fp value, uint64_t error, int& exp, gen_digits_handler& handler) {
   const fp one(1ULL << -value.e, value.e);
   // The integral part of scaled value (p1 in Grisu) = value / one. It cannot be
   // zero because it contains a product of two 64-bit numbers with MSB set (due
@@ -649,10 +700,28 @@ FMT_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, int& exp,
   // The fractional part of scaled value (p2 in Grisu) c = value % one.
   uint64_t fractional = value.f & (one.f - 1);
   exp = count_digits(integral);  // kappa in Grisu.
-  // Divide by 10 to prevent overflow.
-  auto result = handler.on_start(power_of_10_64(exp - 1) << -one.e,
-                                 value.f / 10, error * 10, exp);
-  if (result != digits::more) return result;
+  // Non-fixed formats require at least one digit and no precision adjustment.
+  if (handler.fixed) {
+    // Adjust fixed precision by exponent because it is relative to decimal
+    // point.
+    int precision_offset = exp + handler.exp10;
+    if (precision_offset > 0 &&
+        handler.precision > max_value<int>() - precision_offset) {
+      FMT_THROW(format_error("number is too big"));
+    }
+    handler.precision += precision_offset;
+    // Check if precision is satisfied just by leading zeros, e.g.
+    // format("{:.2f}", 0.001) gives "0.00" without generating any digits.
+    if (handler.precision <= 0) {
+      if (handler.precision < 0) return digits::done;
+      // Divide by 10 to prevent overflow.
+      uint64_t divisor = impl_data::power_of_10_64[exp - 1] << -one.e;
+      auto dir = get_round_direction(divisor, value.f / 10, error * 10);
+      if (dir == round_direction::unknown) return digits::error;
+      handler.buf[handler.size++] = dir == round_direction::up ? '1' : '0';
+      return digits::done;
+    }
+  }
   // Generate digits for the integral part. This can produce up to 10 digits.
   do {
     uint32_t digit = 0;
@@ -699,9 +768,9 @@ FMT_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, int& exp,
     }
     --exp;
     auto remainder = (static_cast<uint64_t>(integral) << -one.e) + fractional;
-    result = handler.on_digit(static_cast<char>('0' + digit),
-                              power_of_10_64(exp) << -one.e, remainder, error,
-                              exp, true);
+    auto result = handler.on_digit(static_cast<char>('0' + digit),
+                                   impl_data::power_of_10_64[exp] << -one.e,
+                                   remainder, error, true);
     if (result != digits::more) return result;
   } while (exp > 0);
   // Generate digits for the fractional part.
@@ -711,69 +780,11 @@ FMT_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, int& exp,
     char digit = static_cast<char>('0' + (fractional >> -one.e));
     fractional &= one.f - 1;
     --exp;
-    result = handler.on_digit(digit, one.f, fractional, error, exp, false);
+    auto result = handler.on_digit(digit, one.f, fractional, error, false);
     if (result != digits::more) return result;
   }
 }
 
-// The fixed precision digit handler.
-struct fixed_handler {
-  char* buf;
-  int size;
-  int precision;
-  int exp10;
-  bool fixed;
-
-  digits::result on_start(uint64_t divisor, uint64_t remainder, uint64_t error,
-                          int& exp) {
-    // Non-fixed formats require at least one digit and no precision adjustment.
-    if (!fixed) return digits::more;
-    // Adjust fixed precision by exponent because it is relative to decimal
-    // point.
-    precision += exp + exp10;
-    // Check if precision is satisfied just by leading zeros, e.g.
-    // format("{:.2f}", 0.001) gives "0.00" without generating any digits.
-    if (precision > 0) return digits::more;
-    if (precision < 0) return digits::done;
-    auto dir = get_round_direction(divisor, remainder, error);
-    if (dir == round_direction::unknown) return digits::error;
-    buf[size++] = dir == round_direction::up ? '1' : '0';
-    return digits::done;
-  }
-
-  digits::result on_digit(char digit, uint64_t divisor, uint64_t remainder,
-                          uint64_t error, int, bool integral) {
-    FMT_ASSERT(remainder < divisor, "");
-    buf[size++] = digit;
-    if (!integral && error >= remainder) return digits::error;
-    if (size < precision) return digits::more;
-    if (!integral) {
-      // Check if error * 2 < divisor with overflow prevention.
-      // The check is not needed for the integral part because error = 1
-      // and divisor > (1 << 32) there.
-      if (error >= divisor || error >= divisor - error) return digits::error;
-    } else {
-      FMT_ASSERT(error == 1 && divisor > 2, "");
-    }
-    auto dir = get_round_direction(divisor, remainder, error);
-    if (dir != round_direction::up)
-      return dir == round_direction::down ? digits::done : digits::error;
-    ++buf[size - 1];
-    for (int i = size - 1; i > 0 && buf[i] > '9'; --i) {
-      buf[i] = '0';
-      ++buf[i - 1];
-    }
-    if (buf[0] > '9') {
-      buf[0] = '1';
-      if (fixed)
-        buf[size++] = '0';
-      else
-        ++exp10;
-    }
-    return digits::done;
-  }
-};
-
 // A 128-bit integer type used internally,
 struct uint128_wrapper {
   uint128_wrapper() = default;
@@ -897,8 +908,7 @@ inline uint64_t umul96_lower64(uint32_t x, uint64_t y) FMT_NOEXCEPT {
 inline int floor_log10_pow2(int e) FMT_NOEXCEPT {
   FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent");
   const int shift = 22;
-  return (e * static_cast<int>(data::log10_2_significand >> (64 - shift))) >>
-         shift;
+  return (e * static_cast<int>(log10_2_significand >> (64 - shift))) >> shift;
 }
 
 // Various fast log computations.
@@ -916,8 +926,7 @@ inline int floor_log10_pow2_minus_log10_4_over_3(int e) FMT_NOEXCEPT {
   FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent");
   const uint64_t log10_4_over_3_fractional_digits = 0x1ffbfc2bbc780375;
   const int shift_amount = 22;
-  return (e * static_cast<int>(data::log10_2_significand >>
-                               (64 - shift_amount)) -
+  return (e * static_cast<int>(log10_2_significand >> (64 - shift_amount)) -
           static_cast<int>(log10_4_over_3_fractional_digits >>
                            (64 - shift_amount))) >>
          shift_amount;
@@ -1042,7 +1051,7 @@ template <> struct cache_accessor<float> {
   static uint64_t get_cached_power(int k) FMT_NOEXCEPT {
     FMT_ASSERT(k >= float_info<float>::min_k && k <= float_info<float>::max_k,
                "k is out of range");
-    constexpr const uint64_t pow10_significands[] = {
+    static constexpr const uint64_t pow10_significands[] = {
         0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f,
         0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb,
         0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28,
@@ -2210,24 +2219,21 @@ small_divisor_case_label:
 }
 }  // namespace dragonbox
 
-// Formats value using a variation of the Fixed-Precision Positive
-// Floating-Point Printout ((FPP)^2) algorithm by Steele & White:
+// Formats a floating-point number using a variation of the Fixed-Precision
+// Positive Floating-Point Printout ((FPP)^2) algorithm by Steele & White:
 // https://fmt.dev/papers/p372-steele.pdf.
-template <typename Double>
-void fallback_format(Double d, int num_digits, bool binary32, buffer<char>& buf,
-                     int& exp10) {
+FMT_CONSTEXPR20 inline void format_dragon(fp value, bool is_predecessor_closer,
+                                          int num_digits, buffer<char>& buf,
+                                          int& exp10) {
   bigint numerator;    // 2 * R in (FPP)^2.
   bigint denominator;  // 2 * S in (FPP)^2.
   // lower and upper are differences between value and corresponding boundaries.
   bigint lower;             // (M^- in (FPP)^2).
   bigint upper_store;       // upper's value if different from lower.
   bigint* upper = nullptr;  // (M^+ in (FPP)^2).
-  fp value;
   // Shift numerator and denominator by an extra bit or two (if lower boundary
   // is closer) to make lower and upper integers. This eliminates multiplication
   // by 2 during later computations.
-  const bool is_predecessor_closer =
-      binary32 ? value.assign(static_cast<float>(d)) : value.assign(d);
   int shift = is_predecessor_closer ? 2 : 1;
   uint64_t significand = value.f << shift;
   if (value.e >= 0) {
@@ -2297,9 +2303,9 @@ void fallback_format(Double d, int num_digits, bool binary32, buffer<char>& buf,
   // Generate the given number of digits.
   exp10 -= num_digits - 1;
   if (num_digits == 0) {
-    buf.try_resize(1);
     denominator *= 10;
-    buf[0] = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0';
+    auto digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0';
+    buf.push_back(digit);
     return;
   }
   buf.try_resize(to_unsigned(num_digits));
@@ -2330,9 +2336,12 @@ void fallback_format(Double d, int num_digits, bool binary32, buffer<char>& buf,
   buf[num_digits - 1] = static_cast<char>('0' + digit);
 }
 
-template <typename T>
-int format_float(T value, int precision, float_specs specs, buffer<char>& buf) {
-  static_assert(!std::is_same<T, float>::value, "");
+template <typename Float>
+FMT_HEADER_ONLY_CONSTEXPR20 int format_float(Float value, int precision,
+                                             float_specs specs,
+                                             buffer<char>& buf) {
+  // float is passed as double to reduce the number of instantiations.
+  static_assert(!std::is_same<Float, float>::value, "");
   FMT_ASSERT(value >= 0, "value is negative");
 
   const bool fixed = specs.format == float_format::fixed;
@@ -2342,13 +2351,13 @@ int format_float(T value, int precision, float_specs specs, buffer<char>& buf) {
       return 0;
     }
     buf.try_resize(to_unsigned(precision));
-    std::uninitialized_fill_n(buf.data(), precision, '0');
+    fill_n(buf.data(), precision, '0');
     return -precision;
   }
 
-  if (!specs.use_grisu) return snprintf_float(value, precision, specs, buf);
+  if (specs.fallback) return snprintf_float(value, precision, specs, buf);
 
-  if (precision < 0) {
+  if (!is_constant_evaluated() && precision < 0) {
     // Use Dragonbox for the shortest format.
     if (specs.binary32) {
       auto dec = dragonbox::to_decimal(static_cast<float>(value));
@@ -2360,26 +2369,37 @@ int format_float(T value, int precision, float_specs specs, buffer<char>& buf) {
     return dec.exponent;
   }
 
-  // Use Grisu + Dragon4 for the given precision:
-  // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf.
   int exp = 0;
-  const int min_exp = -60;  // alpha in Grisu.
-  int cached_exp10 = 0;     // K in Grisu.
-  fp normalized = normalize(fp(value));
-  const auto cached_pow = get_cached_power(
-      min_exp - (normalized.e + fp::significand_size), cached_exp10);
-  normalized = normalized * cached_pow;
-  // Limit precision to the maximum possible number of significant digits in an
-  // IEEE754 double because we don't need to generate zeros.
-  const int max_double_digits = 767;
-  if (precision > max_double_digits) precision = max_double_digits;
-  fixed_handler handler{buf.data(), 0, precision, -cached_exp10, fixed};
-  if (grisu_gen_digits(normalized, 1, exp, handler) == digits::error) {
-    exp += handler.size - cached_exp10 - 1;
-    fallback_format(value, handler.precision, specs.binary32, buf, exp);
-  } else {
-    exp += handler.exp10;
-    buf.try_resize(to_unsigned(handler.size));
+  bool use_dragon = true;
+  if (is_fast_float<Float>()) {
+    // Use Grisu + Dragon4 for the given precision:
+    // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf.
+    const int min_exp = -60;  // alpha in Grisu.
+    int cached_exp10 = 0;     // K in Grisu.
+    fp normalized = normalize(fp(value));
+    const auto cached_pow = get_cached_power(
+        min_exp - (normalized.e + fp::num_significand_bits), cached_exp10);
+    normalized = normalized * cached_pow;
+    gen_digits_handler handler{buf.data(), 0, precision, -cached_exp10, fixed};
+    if (grisu_gen_digits(normalized, 1, exp, handler) != digits::error &&
+        !is_constant_evaluated()) {
+      exp += handler.exp10;
+      buf.try_resize(to_unsigned(handler.size));
+      use_dragon = false;
+    } else {
+      exp += handler.size - cached_exp10 - 1;
+      precision = handler.precision;
+    }
+  }
+  if (use_dragon) {
+    auto f = fp();
+    bool is_predecessor_closer =
+        specs.binary32 ? f.assign(static_cast<float>(value)) : f.assign(value);
+    // Limit precision to the maximum possible number of significant digits in
+    // an IEEE754 double because we don't need to generate zeros.
+    const int max_double_digits = 767;
+    if (precision > max_double_digits) precision = max_double_digits;
+    format_dragon(f, is_predecessor_closer, precision, buf, exp);
   }
   if (!fixed && !specs.showpoint) {
     // Remove trailing zeros.
@@ -2391,7 +2411,7 @@ int format_float(T value, int precision, float_specs specs, buffer<char>& buf) {
     buf.try_resize(num_digits);
   }
   return exp;
-}  // namespace detail
+}
 
 template <typename T>
 int snprintf_float(T value, int precision, float_specs specs,
@@ -2525,8 +2545,8 @@ template <> struct formatter<detail::bigint> {
 };
 
 FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) {
-  for_each_codepoint(s, [this](uint32_t cp, int error) {
-    if (error != 0) FMT_THROW(std::runtime_error("invalid utf8"));
+  for_each_codepoint(s, [this](uint32_t cp, string_view) {
+    if (cp == invalid_code_point) FMT_THROW(std::runtime_error("invalid utf8"));
     if (cp <= 0xFFFF) {
       buffer_.push_back(static_cast<wchar_t>(cp));
     } else {
@@ -2534,6 +2554,7 @@ FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) {
       buffer_.push_back(static_cast<wchar_t>(0xD800 + (cp >> 10)));
       buffer_.push_back(static_cast<wchar_t>(0xDC00 + (cp & 0x3FF)));
     }
+    return true;
   });
   buffer_.push_back(0);
 }
@@ -2549,15 +2570,17 @@ FMT_FUNC void format_system_error(detail::buffer<char>& out, int error_code,
   format_error_code(out, error_code, message);
 }
 
-FMT_FUNC void detail::error_handler::on_error(const char* message) {
-  FMT_THROW(format_error(message));
-}
-
 FMT_FUNC void report_system_error(int error_code,
                                   const char* message) FMT_NOEXCEPT {
   report_error(format_system_error, error_code, message);
 }
 
+// DEPRECATED!
+// This function is defined here and not inline for ABI compatiblity.
+FMT_FUNC void detail::error_handler::on_error(const char* message) {
+  throw_format_error(message);
+}
+
 FMT_FUNC std::string vformat(string_view fmt, format_args args) {
   // Don't optimize the "{}" case to keep the binary size small and because it
   // can be better optimized in fmt::format anyway.
diff --git a/contrib/fmt-8.0.1/include/fmt/format.h b/contrib/fmt-8.1.1/include/fmt/format.h
similarity index 79%
rename from contrib/fmt-8.0.1/include/fmt/format.h
rename to contrib/fmt-8.1.1/include/fmt/format.h
index 5398a23a82..ee69651ca5 100644
--- a/contrib/fmt-8.0.1/include/fmt/format.h
+++ b/contrib/fmt-8.1.1/include/fmt/format.h
@@ -41,14 +41,16 @@
 #include <system_error>  // std::system_error
 #include <utility>       // std::swap
 
+#ifdef __cpp_lib_bit_cast
+#  include <bit>  // std::bitcast
+#endif
+
 #include "core.h"
 
-#ifdef __INTEL_COMPILER
-#  define FMT_ICC_VERSION __INTEL_COMPILER
-#elif defined(__ICL)
-#  define FMT_ICC_VERSION __ICL
+#if FMT_GCC_VERSION
+#  define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
 #else
-#  define FMT_ICC_VERSION 0
+#  define FMT_GCC_VISIBILITY_HIDDEN
 #endif
 
 #ifdef __NVCC__
@@ -108,17 +110,11 @@ FMT_END_NAMESPACE
 #  define FMT_CATCH(x) if (false)
 #endif
 
-#ifndef FMT_DEPRECATED
-#  if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900
-#    define FMT_DEPRECATED [[deprecated]]
+#ifndef FMT_MAYBE_UNUSED
+#  if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused)
+#    define FMT_MAYBE_UNUSED [[maybe_unused]]
 #  else
-#    if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__)
-#      define FMT_DEPRECATED __attribute__((deprecated))
-#    elif FMT_MSC_VER
-#      define FMT_DEPRECATED __declspec(deprecated)
-#    else
-#      define FMT_DEPRECATED /* deprecated */
-#    endif
+#    define FMT_MAYBE_UNUSED
 #  endif
 #endif
 
@@ -149,18 +145,25 @@ FMT_END_NAMESPACE
 #endif
 
 // __builtin_clz is broken in clang with Microsoft CodeGen:
-// https://github.com/fmtlib/fmt/issues/519
-#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clz)) && !FMT_MSC_VER
-#  define FMT_BUILTIN_CLZ(n) __builtin_clz(n)
+// https://github.com/fmtlib/fmt/issues/519.
+#if !FMT_MSC_VER
+#  if FMT_HAS_BUILTIN(__builtin_clz) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CLZ(n) __builtin_clz(n)
+#  endif
+#  if FMT_HAS_BUILTIN(__builtin_clzll) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n)
+#  endif
 #endif
-#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clzll)) && !FMT_MSC_VER
-#  define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n)
-#endif
-#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctz))
-#  define FMT_BUILTIN_CTZ(n) __builtin_ctz(n)
-#endif
-#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctzll))
-#  define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n)
+
+// __builtin_ctz is broken in Intel Compiler Classic on Windows:
+// https://github.com/fmtlib/fmt/issues/2510.
+#ifndef __ICL
+#  if FMT_HAS_BUILTIN(__builtin_ctz) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CTZ(n) __builtin_ctz(n)
+#  endif
+#  if FMT_HAS_BUILTIN(__builtin_ctzll) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n)
+#  endif
 #endif
 
 #if FMT_MSC_VER
@@ -175,7 +178,6 @@ FMT_BEGIN_NAMESPACE
 namespace detail {
 // Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning.
 #  if !defined(__clang__)
-#    pragma managed(push, off)
 #    pragma intrinsic(_BitScanForward)
 #    pragma intrinsic(_BitScanReverse)
 #    if defined(_WIN64)
@@ -237,40 +239,75 @@ inline auto ctzll(uint64_t x) -> int {
   return static_cast<int>(r);
 }
 #  define FMT_BUILTIN_CTZLL(n) detail::ctzll(n)
-#  if !defined(__clang__)
-#    pragma managed(pop)
-#  endif
 }  // namespace detail
 FMT_END_NAMESPACE
 #endif
 
+#ifdef FMT_HEADER_ONLY
+#  define FMT_HEADER_ONLY_CONSTEXPR20 FMT_CONSTEXPR20
+#else
+#  define FMT_HEADER_ONLY_CONSTEXPR20
+#endif
+
 FMT_BEGIN_NAMESPACE
 namespace detail {
 
-#if __cplusplus >= 202002L || \
-    (__cplusplus >= 201709L && FMT_GCC_VERSION >= 1002)
-#  define FMT_CONSTEXPR20 constexpr
-#else
-#  define FMT_CONSTEXPR20
-#endif
+template <typename Streambuf> class formatbuf : public Streambuf {
+ private:
+  using char_type = typename Streambuf::char_type;
+  using streamsize = decltype(std::declval<Streambuf>().sputn(nullptr, 0));
+  using int_type = typename Streambuf::int_type;
+  using traits_type = typename Streambuf::traits_type;
 
-// An equivalent of `*reinterpret_cast<Dest*>(&source)` that doesn't have
-// undefined behavior (e.g. due to type aliasing).
-// Example: uint64_t d = bit_cast<uint64_t>(2.718);
-template <typename Dest, typename Source>
-inline auto bit_cast(const Source& source) -> Dest {
-  static_assert(sizeof(Dest) == sizeof(Source), "size mismatch");
-  Dest dest;
-  std::memcpy(&dest, &source, sizeof(dest));
-  return dest;
+  buffer<char_type>& buffer_;
+
+ public:
+  explicit formatbuf(buffer<char_type>& buf) : buffer_(buf) {}
+
+ protected:
+  // The put area is always empty. This makes the implementation simpler and has
+  // the advantage that the streambuf and the buffer are always in sync and
+  // sputc never writes into uninitialized memory. A disadvantage is that each
+  // call to sputc always results in a (virtual) call to overflow. There is no
+  // disadvantage here for sputn since this always results in a call to xsputn.
+
+  auto overflow(int_type ch) -> int_type override {
+    if (!traits_type::eq_int_type(ch, traits_type::eof()))
+      buffer_.push_back(static_cast<char_type>(ch));
+    return ch;
+  }
+
+  auto xsputn(const char_type* s, streamsize count) -> streamsize override {
+    buffer_.append(s, s + count);
+    return count;
+  }
+};
+
+// Implementation of std::bit_cast for pre-C++20.
+template <typename To, typename From>
+FMT_CONSTEXPR20 auto bit_cast(const From& from) -> To {
+  static_assert(sizeof(To) == sizeof(From), "size mismatch");
+#ifdef __cpp_lib_bit_cast
+  if (is_constant_evaluated()) return std::bit_cast<To>(from);
+#endif
+  auto to = To();
+  std::memcpy(&to, &from, sizeof(to));
+  return to;
 }
 
 inline auto is_big_endian() -> bool {
-  const auto u = 1u;
+#ifdef _WIN32
+  return false;
+#elif defined(__BIG_ENDIAN__)
+  return true;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)
+  return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
+#else
   struct bytes {
-    char data[sizeof(u)];
+    char data[sizeof(int)];
   };
-  return bit_cast<bytes>(u).data[0] == 0;
+  return bit_cast<bytes>(1).data[0] == 0;
+#endif
 }
 
 // A fallback implementation of uintptr_t for systems that lack it.
@@ -280,7 +317,7 @@ struct fallback_uintptr {
   fallback_uintptr() = default;
   explicit fallback_uintptr(const void* p) {
     *this = bit_cast<fallback_uintptr>(p);
-    if (is_big_endian()) {
+    if (const_check(is_big_endian())) {
       for (size_t i = 0, j = sizeof(void*) - 1; i < j; ++i, --j)
         std::swap(value[i], value[j]);
     }
@@ -339,12 +376,15 @@ inline auto get_data(Container& c) -> typename Container::value_type* {
 #if defined(_SECURE_SCL) && _SECURE_SCL
 // Make a checked iterator to avoid MSVC warnings.
 template <typename T> using checked_ptr = stdext::checked_array_iterator<T*>;
-template <typename T> auto make_checked(T* p, size_t size) -> checked_ptr<T> {
+template <typename T>
+constexpr auto make_checked(T* p, size_t size) -> checked_ptr<T> {
   return {p, size};
 }
 #else
 template <typename T> using checked_ptr = T*;
-template <typename T> inline auto make_checked(T* p, size_t) -> T* { return p; }
+template <typename T> constexpr auto make_checked(T* p, size_t) -> T* {
+  return p;
+}
 #endif
 
 // Attempts to reserve space for n extra characters in the output range.
@@ -480,27 +520,38 @@ FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
   return next;
 }
 
+constexpr uint32_t invalid_code_point = ~uint32_t();
+
+// Invokes f(cp, sv) for every code point cp in s with sv being the string view
+// corresponding to the code point. cp is invalid_code_point on error.
 template <typename F>
 FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) {
-  auto decode = [f](const char* p) {
+  auto decode = [f](const char* buf_ptr, const char* ptr) {
     auto cp = uint32_t();
     auto error = 0;
-    p = utf8_decode(p, &cp, &error);
-    f(cp, error);
-    return p;
+    auto end = utf8_decode(buf_ptr, &cp, &error);
+    bool result = f(error ? invalid_code_point : cp,
+                    string_view(ptr, to_unsigned(end - buf_ptr)));
+    return result ? end : nullptr;
   };
   auto p = s.data();
   const size_t block_size = 4;  // utf8_decode always reads blocks of 4 chars.
   if (s.size() >= block_size) {
-    for (auto end = p + s.size() - block_size + 1; p < end;) p = decode(p);
+    for (auto end = p + s.size() - block_size + 1; p < end;) {
+      p = decode(p, p);
+      if (!p) return;
+    }
   }
   if (auto num_chars_left = s.data() + s.size() - p) {
     char buf[2 * block_size - 1] = {};
     copy_str<char>(p, p + num_chars_left, buf);
-    p = buf;
+    const char* buf_ptr = buf;
     do {
-      p = decode(p);
-    } while (p - buf < num_chars_left);
+      auto end = decode(buf_ptr, p);
+      if (!end) return;
+      p += end - buf_ptr;
+      buf_ptr = end;
+    } while (buf_ptr - buf < num_chars_left);
   }
 }
 
@@ -515,10 +566,10 @@ FMT_CONSTEXPR inline size_t compute_width(string_view s) {
   // It is not a lambda for compatibility with C++14.
   struct count_code_points {
     size_t* count;
-    FMT_CONSTEXPR void operator()(uint32_t cp, int error) const {
+    FMT_CONSTEXPR auto operator()(uint32_t cp, string_view) const -> bool {
       *count += detail::to_unsigned(
           1 +
-          (error == 0 && cp >= 0x1100 &&
+          (cp >= 0x1100 &&
            (cp <= 0x115f ||  // Hangul Jamo init. consonants
             cp == 0x2329 ||  // LEFT-POINTING ANGLE BRACKET
             cp == 0x232a ||  // RIGHT-POINTING ANGLE BRACKET
@@ -536,6 +587,7 @@ FMT_CONSTEXPR inline size_t compute_width(string_view s) {
             (cp >= 0x1f300 && cp <= 0x1f64f) ||
             // Supplemental Symbols and Pictographs:
             (cp >= 0x1f900 && cp <= 0x1f9ff))));
+      return true;
     }
   };
   for_each_codepoint(s, count_code_points{&num_code_points});
@@ -564,9 +616,10 @@ inline auto code_point_index(basic_string_view<char8_type> s, size_t n)
   return s.size();
 }
 
-template <typename T>
-using is_fast_float = bool_constant<std::numeric_limits<T>::is_iec559 &&
-                                    sizeof(T) <= sizeof(double)>;
+template <typename T, bool = std::is_floating_point<T>::value>
+struct is_fast_float : bool_constant<std::numeric_limits<T>::is_iec559 &&
+                                     sizeof(T) <= sizeof(double)> {};
+template <typename T> struct is_fast_float<T, false> : std::false_type {};
 
 #ifndef FMT_USE_FULL_CACHE_DRAGONBOX
 #  define FMT_USE_FULL_CACHE_DRAGONBOX 0
@@ -607,8 +660,8 @@ enum { inline_buffer_size = 500 };
 
   **Example**::
 
-     fmt::memory_buffer out;
-     format_to(out, "The answer is {}.", 42);
+     auto out = fmt::memory_buffer();
+     format_to(std::back_inserter(out), "The answer is {}.", 42);
 
   This will append the following output to the ``out`` object:
 
@@ -629,34 +682,43 @@ class basic_memory_buffer final : public detail::buffer<T> {
   Allocator alloc_;
 
   // Deallocate memory allocated by the buffer.
-  void deallocate() {
+  FMT_CONSTEXPR20 void deallocate() {
     T* data = this->data();
     if (data != store_) alloc_.deallocate(data, this->capacity());
   }
 
  protected:
-  void grow(size_t size) final FMT_OVERRIDE;
+  FMT_CONSTEXPR20 void grow(size_t size) override;
 
  public:
   using value_type = T;
   using const_reference = const T&;
 
-  explicit basic_memory_buffer(const Allocator& alloc = Allocator())
+  FMT_CONSTEXPR20 explicit basic_memory_buffer(
+      const Allocator& alloc = Allocator())
       : alloc_(alloc) {
     this->set(store_, SIZE);
+    if (detail::is_constant_evaluated()) {
+      detail::fill_n(store_, SIZE, T{});
+    }
   }
-  ~basic_memory_buffer() { deallocate(); }
+  FMT_CONSTEXPR20 ~basic_memory_buffer() { deallocate(); }
 
  private:
   // Move data from other to this buffer.
-  void move(basic_memory_buffer& other) {
+  FMT_CONSTEXPR20 void move(basic_memory_buffer& other) {
     alloc_ = std::move(other.alloc_);
     T* data = other.data();
     size_t size = other.size(), capacity = other.capacity();
     if (data == other.store_) {
       this->set(store_, capacity);
-      std::uninitialized_copy(other.store_, other.store_ + size,
-                              detail::make_checked(store_, capacity));
+      if (detail::is_constant_evaluated()) {
+        detail::copy_str<T>(other.store_, other.store_ + size,
+                            detail::make_checked(store_, capacity));
+      } else {
+        std::uninitialized_copy(other.store_, other.store_ + size,
+                                detail::make_checked(store_, capacity));
+      }
     } else {
       this->set(data, capacity);
       // Set pointer to the inline array so that delete is not called
@@ -673,7 +735,10 @@ class basic_memory_buffer final : public detail::buffer<T> {
     of the other object to it.
     \endrst
    */
-  basic_memory_buffer(basic_memory_buffer&& other) FMT_NOEXCEPT { move(other); }
+  FMT_CONSTEXPR20 basic_memory_buffer(basic_memory_buffer&& other)
+      FMT_NOEXCEPT {
+    move(other);
+  }
 
   /**
     \rst
@@ -695,7 +760,7 @@ class basic_memory_buffer final : public detail::buffer<T> {
     Resizes the buffer to contain *count* elements. If T is a POD type new
     elements may not be initialized.
    */
-  void resize(size_t count) { this->try_resize(count); }
+  FMT_CONSTEXPR20 void resize(size_t count) { this->try_resize(count); }
 
   /** Increases the buffer capacity to *new_capacity*. */
   void reserve(size_t new_capacity) { this->try_reserve(new_capacity); }
@@ -709,7 +774,8 @@ class basic_memory_buffer final : public detail::buffer<T> {
 };
 
 template <typename T, size_t SIZE, typename Allocator>
-void basic_memory_buffer<T, SIZE, Allocator>::grow(size_t size) {
+FMT_CONSTEXPR20 void basic_memory_buffer<T, SIZE, Allocator>::grow(
+    size_t size) {
 #ifdef FMT_FUZZ
   if (size > 5000) throw std::runtime_error("fuzz mode - won't grow that much");
 #endif
@@ -754,7 +820,7 @@ class FMT_API format_error : public std::runtime_error {
   format_error& operator=(const format_error&) = default;
   format_error(format_error&&) = default;
   format_error& operator=(format_error&&) = default;
-  ~format_error() FMT_NOEXCEPT FMT_OVERRIDE FMT_MSC_DEFAULT;
+  ~format_error() FMT_NOEXCEPT override FMT_MSC_DEFAULT;
 };
 
 /**
@@ -807,10 +873,6 @@ constexpr auto compile_string_to_view(detail::std_string_view<Char> s)
 
 FMT_BEGIN_DETAIL_NAMESPACE
 
-inline void throw_format_error(const char* message) {
-  FMT_THROW(format_error(message));
-}
-
 template <typename T> struct is_integral : std::is_integral<T> {};
 template <> struct is_integral<int128_t> : std::true_type {};
 template <> struct is_integral<uint128_t> : std::true_type {};
@@ -853,48 +915,23 @@ using uint64_or_128_t = conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>;
       (factor)*1000000, (factor)*10000000, (factor)*100000000,               \
       (factor)*1000000000
 
-// Static data is placed in this class template for the header-only config.
-template <typename T = void> struct basic_data {
-  // log10(2) = 0x0.4d104d427de7fbcc...
-  static const uint64_t log10_2_significand = 0x4d104d427de7fbcc;
+// Converts value in the range [0, 100) to a string.
+constexpr const char* digits2(size_t value) {
+  // GCC generates slightly better code when value is pointer-size.
+  return &"0001020304050607080910111213141516171819"
+         "2021222324252627282930313233343536373839"
+         "4041424344454647484950515253545556575859"
+         "6061626364656667686970717273747576777879"
+         "8081828384858687888990919293949596979899"[value * 2];
+}
 
-  // GCC generates slightly better code for pairs than chars.
-  FMT_API static constexpr const char digits[100][2] = {
-      {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'},
-      {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'},
-      {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'},
-      {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'},
-      {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'},
-      {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'},
-      {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'},
-      {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'},
-      {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'},
-      {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'},
-      {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'},
-      {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'},
-      {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'},
-      {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'},
-      {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'},
-      {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'},
-      {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}};
-
-  FMT_API static constexpr const char hex_digits[] = "0123456789abcdef";
-  FMT_API static constexpr const char signs[4] = {0, '-', '+', ' '};
-  FMT_API static constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
-                                                         0x1000000u | ' '};
-  FMT_API static constexpr const char left_padding_shifts[5] = {31, 31, 0, 1,
-                                                                0};
-  FMT_API static constexpr const char right_padding_shifts[5] = {0, 31, 0, 1,
-                                                                 0};
-};
-
-#ifdef FMT_SHARED
-// Required for -flto, -fivisibility=hidden and -shared to work
-extern template struct basic_data<void>;
+// Sign is a template parameter to workaround a bug in gcc 4.8.
+template <typename Char, typename Sign> constexpr Char sign(Sign s) {
+#if !FMT_GCC_VERSION || FMT_GCC_VERSION >= 604
+  static_assert(std::is_same<Sign, sign_t>::value, "");
 #endif
-
-// This is a struct rather than an alias to avoid shadowing warnings in gcc.
-struct data : basic_data<> {};
+  return static_cast<Char>("\0-+ "[s]);
+}
 
 template <typename T> FMT_CONSTEXPR auto count_digits_fallback(T n) -> int {
   int count = 1;
@@ -916,23 +953,33 @@ FMT_CONSTEXPR inline auto count_digits(uint128_t n) -> int {
 }
 #endif
 
+#ifdef FMT_BUILTIN_CLZLL
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+inline auto do_count_digits(uint64_t n) -> int {
+  // This has comparable performance to the version by Kendall Willets
+  // (https://github.com/fmtlib/format-benchmark/blob/master/digits10)
+  // but uses smaller tables.
+  // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)).
+  static constexpr uint8_t bsr2log10[] = {
+      1,  1,  1,  2,  2,  2,  3,  3,  3,  4,  4,  4,  4,  5,  5,  5,
+      6,  6,  6,  7,  7,  7,  7,  8,  8,  8,  9,  9,  9,  10, 10, 10,
+      10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15,
+      15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20};
+  auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63];
+  static constexpr const uint64_t zero_or_powers_of_10[] = {
+      0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL),
+      10000000000000000000ULL};
+  return t - (n < zero_or_powers_of_10[t]);
+}
+#endif
+
 // Returns the number of decimal digits in n. Leading zeros are not counted
 // except for n == 0 in which case count_digits returns 1.
 FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int {
 #ifdef FMT_BUILTIN_CLZLL
   if (!is_constant_evaluated()) {
-    // https://github.com/fmtlib/format-benchmark/blob/master/digits10
-    // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)).
-    constexpr uint16_t bsr2log10[] = {
-        1,  1,  1,  2,  2,  2,  3,  3,  3,  4,  4,  4,  4,  5,  5,  5,
-        6,  6,  6,  7,  7,  7,  7,  8,  8,  8,  9,  9,  9,  10, 10, 10,
-        10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15,
-        15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20};
-    auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63];
-    constexpr const uint64_t zero_or_powers_of_10[] = {
-        0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL),
-        10000000000000000000ULL};
-    return t - (n < zero_or_powers_of_10[t]);
+    return do_count_digits(n);
   }
 #endif
   return count_digits_fallback(n);
@@ -945,21 +992,25 @@ FMT_CONSTEXPR auto count_digits(UInt n) -> int {
   if (num_bits<UInt>() == 32)
     return (FMT_BUILTIN_CLZ(static_cast<uint32_t>(n) | 1) ^ 31) / BITS + 1;
 #endif
-  int num_digits = 0;
-  do {
-    ++num_digits;
-  } while ((n >>= BITS) != 0);
-  return num_digits;
+  // Lambda avoids unreachable code warnings from NVHPC.
+  return [](UInt m) {
+    int num_digits = 0;
+    do {
+      ++num_digits;
+    } while ((m >>= BITS) != 0);
+    return num_digits;
+  }(n);
 }
 
 template <> auto count_digits<4>(detail::fallback_uintptr n) -> int;
 
+#ifdef FMT_BUILTIN_CLZ
 // It is a separate function rather than a part of count_digits to workaround
 // the lack of static constexpr in constexpr functions.
-FMT_INLINE uint64_t count_digits_inc(int n) {
-  // An optimization by Kendall Willets from https://bit.ly/3uOIQrB.
-  // This increments the upper 32 bits (log10(T) - 1) when >= T is added.
-#define FMT_INC(T) (((sizeof(#T) - 1ull) << 32) - T)
+FMT_INLINE auto do_count_digits(uint32_t n) -> int {
+// An optimization by Kendall Willets from https://bit.ly/3uOIQrB.
+// This increments the upper 32 bits (log10(T) - 1) when >= T is added.
+#  define FMT_INC(T) (((sizeof(#  T) - 1ull) << 32) - T)
   static constexpr uint64_t table[] = {
       FMT_INC(0),          FMT_INC(0),          FMT_INC(0),           // 8
       FMT_INC(10),         FMT_INC(10),         FMT_INC(10),          // 64
@@ -973,15 +1024,16 @@ FMT_INLINE uint64_t count_digits_inc(int n) {
       FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000),  // 1024M
       FMT_INC(1000000000), FMT_INC(1000000000)                        // 4B
   };
-  return table[n];
+  auto inc = table[FMT_BUILTIN_CLZ(n | 1) ^ 31];
+  return static_cast<int>((n + inc) >> 32);
 }
+#endif
 
 // Optional version of count_digits for better performance on 32-bit platforms.
 FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int {
 #ifdef FMT_BUILTIN_CLZ
   if (!is_constant_evaluated()) {
-    auto inc = count_digits_inc(FMT_BUILTIN_CLZ(n | 1) ^ 31);
-    return static_cast<int>((n + inc) >> 32);
+    return do_count_digits(n);
   }
 #endif
   return count_digits_fallback(n);
@@ -1032,11 +1084,15 @@ inline auto equal2(const char* lhs, const char* rhs) -> bool {
 }
 
 // Copies two characters from src to dst.
-template <typename Char> void copy2(Char* dst, const char* src) {
+template <typename Char>
+FMT_CONSTEXPR20 FMT_INLINE void copy2(Char* dst, const char* src) {
+  if (!is_constant_evaluated() && sizeof(Char) == sizeof(char)) {
+    memcpy(dst, src, 2);
+    return;
+  }
   *dst++ = static_cast<Char>(*src++);
   *dst = static_cast<Char>(*src);
 }
-FMT_INLINE void copy2(char* dst, const char* src) { memcpy(dst, src, 2); }
 
 template <typename Iterator> struct format_decimal_result {
   Iterator begin;
@@ -1052,20 +1108,12 @@ FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size)
   FMT_ASSERT(size >= count_digits(value), "invalid digit count");
   out += size;
   Char* end = out;
-  if (is_constant_evaluated()) {
-    while (value >= 10) {
-      *--out = static_cast<Char>('0' + value % 10);
-      value /= 10;
-    }
-    *--out = static_cast<Char>('0' + value);
-    return {out, end};
-  }
   while (value >= 100) {
     // Integer division is slow so do it for a group of two digits instead
     // of for every digit. The idea comes from the talk by Alexandrescu
     // "Three Optimization Tips for C++". See speed-test for a comparison.
     out -= 2;
-    copy2(out, data::digits[value % 100]);
+    copy2(out, digits2(static_cast<size_t>(value % 100)));
     value /= 100;
   }
   if (value < 10) {
@@ -1073,7 +1121,7 @@ FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size)
     return {out, end};
   }
   out -= 2;
-  copy2(out, data::digits[value]);
+  copy2(out, digits2(static_cast<size_t>(value)));
   return {out, end};
 }
 
@@ -1093,7 +1141,7 @@ FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits,
   buffer += num_digits;
   Char* end = buffer;
   do {
-    const char* digits = upper ? "0123456789ABCDEF" : data::hex_digits;
+    const char* digits = upper ? "0123456789ABCDEF" : "0123456789abcdef";
     unsigned digit = (value & ((1 << BASE_BITS) - 1));
     *--buffer = static_cast<Char>(BASE_BITS < 4 ? static_cast<char>('0' + digit)
                                                 : digits[digit]);
@@ -1116,7 +1164,7 @@ auto format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits,
     auto p = buffer;
     for (int i = 0; i < char_digits; ++i) {
       unsigned digit = (value & ((1 << BASE_BITS) - 1));
-      *--p = static_cast<Char>(data::hex_digits[digit]);
+      *--p = static_cast<Char>("0123456789abcdef"[digit]);
       value >>= BASE_BITS;
     }
   }
@@ -1226,7 +1274,7 @@ constexpr auto exponent_mask() ->
 
 // Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
 template <typename Char, typename It>
-auto write_exponent(int exp, It it) -> It {
+FMT_CONSTEXPR auto write_exponent(int exp, It it) -> It {
   FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range");
   if (exp < 0) {
     *it++ = static_cast<Char>('-');
@@ -1235,28 +1283,31 @@ auto write_exponent(int exp, It it) -> It {
     *it++ = static_cast<Char>('+');
   }
   if (exp >= 100) {
-    const char* top = data::digits[exp / 100];
+    const char* top = digits2(to_unsigned(exp / 100));
     if (exp >= 1000) *it++ = static_cast<Char>(top[0]);
     *it++ = static_cast<Char>(top[1]);
     exp %= 100;
   }
-  const char* d = data::digits[exp];
+  const char* d = digits2(to_unsigned(exp));
   *it++ = static_cast<Char>(d[0]);
   *it++ = static_cast<Char>(d[1]);
   return it;
 }
 
 template <typename T>
-auto format_float(T value, int precision, float_specs specs, buffer<char>& buf)
-    -> int;
+FMT_HEADER_ONLY_CONSTEXPR20 auto format_float(T value, int precision,
+                                              float_specs specs,
+                                              buffer<char>& buf) -> int;
 
 // Formats a floating-point number with snprintf.
 template <typename T>
 auto snprintf_float(T value, int precision, float_specs specs,
                     buffer<char>& buf) -> int;
 
-template <typename T> auto promote_float(T value) -> T { return value; }
-inline auto promote_float(float value) -> double {
+template <typename T> constexpr auto promote_float(T value) -> T {
+  return value;
+}
+constexpr auto promote_float(float value) -> double {
   return static_cast<double>(value);
 }
 
@@ -1282,8 +1333,9 @@ FMT_CONSTEXPR auto write_padded(OutputIt out,
   static_assert(align == align::left || align == align::right, "");
   unsigned spec_width = to_unsigned(specs.width);
   size_t padding = spec_width > width ? spec_width - width : 0;
-  auto* shifts = align == align::left ? data::left_padding_shifts
-                                      : data::right_padding_shifts;
+  // Shifts are encoded as string literals because static constexpr is not
+  // supported in constexpr functions.
+  auto* shifts = align == align::left ? "\x1f\x1f\x00\x01" : "\x00\x1f\x00\x01";
   size_t left_padding = padding >> shifts[specs.align];
   size_t right_padding = padding - left_padding;
   auto it = reserve(out, size + padding * specs.fill.size());
@@ -1393,56 +1445,91 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits,
       });
 }
 
+template <typename Char> class digit_grouping {
+ private:
+  thousands_sep_result<Char> sep_;
+
+  struct next_state {
+    std::string::const_iterator group;
+    int pos;
+  };
+  next_state initial_state() const { return {sep_.grouping.begin(), 0}; }
+
+  // Returns the next digit group separator position.
+  int next(next_state& state) const {
+    if (!sep_.thousands_sep) return max_value<int>();
+    if (state.group == sep_.grouping.end())
+      return state.pos += sep_.grouping.back();
+    if (*state.group <= 0 || *state.group == max_value<char>())
+      return max_value<int>();
+    state.pos += *state.group++;
+    return state.pos;
+  }
+
+ public:
+  explicit digit_grouping(locale_ref loc, bool localized = true) {
+    if (localized)
+      sep_ = thousands_sep<Char>(loc);
+    else
+      sep_.thousands_sep = Char();
+  }
+  explicit digit_grouping(thousands_sep_result<Char> sep) : sep_(sep) {}
+
+  Char separator() const { return sep_.thousands_sep; }
+
+  int count_separators(int num_digits) const {
+    int count = 0;
+    auto state = initial_state();
+    while (num_digits > next(state)) ++count;
+    return count;
+  }
+
+  // Applies grouping to digits and write the output to out.
+  template <typename Out, typename C>
+  Out apply(Out out, basic_string_view<C> digits) const {
+    auto num_digits = static_cast<int>(digits.size());
+    auto separators = basic_memory_buffer<int>();
+    separators.push_back(0);
+    auto state = initial_state();
+    while (int i = next(state)) {
+      if (i >= num_digits) break;
+      separators.push_back(i);
+    }
+    for (int i = 0, sep_index = static_cast<int>(separators.size() - 1);
+         i < num_digits; ++i) {
+      if (num_digits - i == separators[sep_index]) {
+        *out++ = separator();
+        --sep_index;
+      }
+      *out++ = static_cast<Char>(digits[to_unsigned(i)]);
+    }
+    return out;
+  }
+};
+
+template <typename OutputIt, typename UInt, typename Char>
+auto write_int_localized(OutputIt out, UInt value, unsigned prefix,
+                         const basic_format_specs<Char>& specs,
+                         const digit_grouping<Char>& grouping) -> OutputIt {
+  static_assert(std::is_same<uint64_or_128_t<UInt>, UInt>::value, "");
+  int num_digits = count_digits(value);
+  char digits[40];
+  format_decimal(digits, value, num_digits);
+  unsigned size = to_unsigned((prefix != 0 ? 1 : 0) + num_digits +
+                              grouping.count_separators(num_digits));
+  return write_padded<align::right>(
+      out, specs, size, size, [&](reserve_iterator<OutputIt> it) {
+        if (prefix != 0) *it++ = static_cast<Char>(prefix);
+        return grouping.apply(it, string_view(digits, to_unsigned(num_digits)));
+      });
+}
+
 template <typename OutputIt, typename UInt, typename Char>
 auto write_int_localized(OutputIt& out, UInt value, unsigned prefix,
                          const basic_format_specs<Char>& specs, locale_ref loc)
     -> bool {
-  static_assert(std::is_same<uint64_or_128_t<UInt>, UInt>::value, "");
-  const auto sep_size = 1;
-  auto ts = thousands_sep<Char>(loc);
-  if (!ts.thousands_sep) return false;
-  int num_digits = count_digits(value);
-  int size = num_digits, n = num_digits;
-  const std::string& groups = ts.grouping;
-  std::string::const_iterator group = groups.cbegin();
-  while (group != groups.cend() && n > *group && *group > 0 &&
-         *group != max_value<char>()) {
-    size += sep_size;
-    n -= *group;
-    ++group;
-  }
-  if (group == groups.cend()) size += sep_size * ((n - 1) / groups.back());
-  char digits[40];
-  format_decimal(digits, value, num_digits);
-  basic_memory_buffer<Char> buffer;
-  if (prefix != 0) ++size;
-  const auto usize = to_unsigned(size);
-  buffer.resize(usize);
-  basic_string_view<Char> s(&ts.thousands_sep, sep_size);
-  // Index of a decimal digit with the least significant digit having index 0.
-  int digit_index = 0;
-  group = groups.cbegin();
-  auto p = buffer.data() + size - 1;
-  for (int i = num_digits - 1; i > 0; --i) {
-    *p-- = static_cast<Char>(digits[i]);
-    if (*group <= 0 || ++digit_index % *group != 0 ||
-        *group == max_value<char>())
-      continue;
-    if (group + 1 != groups.cend()) {
-      digit_index = 0;
-      ++group;
-    }
-    std::uninitialized_copy(s.data(), s.data() + s.size(),
-                            make_checked(p, s.size()));
-    p -= s.size();
-  }
-  *p-- = static_cast<Char>(*digits);
-  if (prefix != 0) *p = static_cast<Char>(prefix);
-  auto data = buffer.data();
-  out = write_padded<align::right>(
-      out, specs, usize, usize, [=](reserve_iterator<OutputIt> it) {
-        return copy_str<Char>(data, data + size, it);
-      });
+  auto grouping = digit_grouping<Char>(loc);
+  out = write_int_localized(out, value, prefix, specs, grouping);
   return true;
 }
 
@@ -1465,7 +1552,9 @@ FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign)
     prefix = 0x01000000 | '-';
     abs_value = 0 - abs_value;
   } else {
-    prefix = data::prefixes[sign];
+    constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
+                                            0x1000000u | ' '};
+    prefix = prefixes[sign];
   }
   return {abs_value, prefix};
 }
@@ -1477,10 +1566,9 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
   static_assert(std::is_same<T, uint32_or_64_or_128_t<T>>::value, "");
   auto abs_value = arg.abs_value;
   auto prefix = arg.prefix;
-  auto utype = static_cast<unsigned>(specs.type);
   switch (specs.type) {
-  case 0:
-  case 'd': {
+  case presentation_type::none:
+  case presentation_type::dec: {
     if (specs.localized &&
         write_int_localized(out, static_cast<uint64_or_128_t<T>>(abs_value),
                             prefix, specs, loc)) {
@@ -1492,52 +1580,61 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
           return format_decimal<Char>(it, abs_value, num_digits).end;
         });
   }
-  case 'x':
-  case 'X': {
-    if (specs.alt) prefix_append(prefix, (utype << 8) | '0');
-    bool upper = specs.type != 'x';
+  case presentation_type::hex_lower:
+  case presentation_type::hex_upper: {
+    bool upper = specs.type == presentation_type::hex_upper;
+    if (specs.alt)
+      prefix_append(prefix, unsigned(upper ? 'X' : 'x') << 8 | '0');
     int num_digits = count_digits<4>(abs_value);
     return write_int(
         out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
           return format_uint<4, Char>(it, abs_value, num_digits, upper);
         });
   }
-  case 'b':
-  case 'B': {
-    if (specs.alt) prefix_append(prefix, (utype << 8) | '0');
+  case presentation_type::bin_lower:
+  case presentation_type::bin_upper: {
+    bool upper = specs.type == presentation_type::bin_upper;
+    if (specs.alt)
+      prefix_append(prefix, unsigned(upper ? 'B' : 'b') << 8 | '0');
     int num_digits = count_digits<1>(abs_value);
     return write_int(out, num_digits, prefix, specs,
                      [=](reserve_iterator<OutputIt> it) {
                        return format_uint<1, Char>(it, abs_value, num_digits);
                      });
   }
-  case 'o': {
+  case presentation_type::oct: {
     int num_digits = count_digits<3>(abs_value);
-    if (specs.alt && specs.precision <= num_digits && abs_value != 0) {
-      // Octal prefix '0' is counted as a digit, so only add it if precision
-      // is not greater than the number of digits.
+    // Octal prefix '0' is counted as a digit, so only add it if precision
+    // is not greater than the number of digits.
+    if (specs.alt && specs.precision <= num_digits && abs_value != 0)
       prefix_append(prefix, '0');
-    }
     return write_int(out, num_digits, prefix, specs,
                      [=](reserve_iterator<OutputIt> it) {
                        return format_uint<3, Char>(it, abs_value, num_digits);
                      });
   }
-  case 'c':
+  case presentation_type::chr:
     return write_char(out, static_cast<Char>(abs_value), specs);
   default:
-    FMT_THROW(format_error("invalid type specifier"));
+    throw_format_error("invalid type specifier");
   }
   return out;
 }
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline(
+    OutputIt out, write_int_arg<T> arg, const basic_format_specs<Char>& specs,
+    locale_ref loc) -> OutputIt {
+  return write_int(out, arg, specs, loc);
+}
 template <typename Char, typename OutputIt, typename T,
           FMT_ENABLE_IF(is_integral<T>::value &&
                         !std::is_same<T, bool>::value &&
                         std::is_same<OutputIt, buffer_appender<Char>>::value)>
-FMT_CONSTEXPR auto write(OutputIt out, T value,
-                         const basic_format_specs<Char>& specs, locale_ref loc)
-    -> OutputIt {
-  return write_int(out, make_write_int_arg(value, specs.sign), specs, loc);
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+                                    const basic_format_specs<Char>& specs,
+                                    locale_ref loc) -> OutputIt {
+  return write_int_noinline(out, make_write_int_arg(value, specs.sign), specs,
+                            loc);
 }
 // An inlined version of write used in format string compilation.
 template <typename Char, typename OutputIt, typename T,
@@ -1582,8 +1679,9 @@ FMT_CONSTEXPR auto write(OutputIt out, const Char* s,
 }
 
 template <typename Char, typename OutputIt>
-auto write_nonfinite(OutputIt out, bool isinf, basic_format_specs<Char> specs,
-                     const float_specs& fspecs) -> OutputIt {
+FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isinf,
+                                     basic_format_specs<Char> specs,
+                                     const float_specs& fspecs) -> OutputIt {
   auto str =
       isinf ? (fspecs.upper ? "INF" : "inf") : (fspecs.upper ? "NAN" : "nan");
   constexpr size_t str_size = 3;
@@ -1594,7 +1692,7 @@ auto write_nonfinite(OutputIt out, bool isinf, basic_format_specs<Char> specs,
       specs.fill.size() == 1 && *specs.fill.data() == static_cast<Char>('0');
   if (is_zero_fill) specs.fill[0] = static_cast<Char>(' ');
   return write_padded(out, specs, size, [=](reserve_iterator<OutputIt> it) {
-    if (sign) *it++ = static_cast<Char>(data::signs[sign]);
+    if (sign) *it++ = detail::sign<Char>(sign);
     return copy_str<Char>(str, str + str_size, it);
   });
 }
@@ -1606,7 +1704,7 @@ struct big_decimal_fp {
   int exponent;
 };
 
-inline auto get_significand_size(const big_decimal_fp& fp) -> int {
+constexpr auto get_significand_size(const big_decimal_fp& fp) -> int {
   return fp.significand_size;
 }
 template <typename T>
@@ -1615,8 +1713,8 @@ inline auto get_significand_size(const dragonbox::decimal_fp<T>& fp) -> int {
 }
 
 template <typename Char, typename OutputIt>
-inline auto write_significand(OutputIt out, const char* significand,
-                              int& significand_size) -> OutputIt {
+constexpr auto write_significand(OutputIt out, const char* significand,
+                                 int significand_size) -> OutputIt {
   return copy_str<Char>(significand, significand + significand_size, out);
 }
 template <typename Char, typename OutputIt, typename UInt>
@@ -1624,6 +1722,19 @@ inline auto write_significand(OutputIt out, UInt significand,
                               int significand_size) -> OutputIt {
   return format_decimal<Char>(out, significand, significand_size).end;
 }
+template <typename Char, typename OutputIt, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+                                       int significand_size, int exponent,
+                                       const Grouping& grouping) -> OutputIt {
+  if (!grouping.separator()) {
+    out = write_significand<Char>(out, significand, significand_size);
+    return detail::fill_n(out, exponent, static_cast<Char>('0'));
+  }
+  auto buffer = memory_buffer();
+  write_significand<char>(appender(buffer), significand, significand_size);
+  detail::fill_n(appender(buffer), exponent, '0');
+  return grouping.apply(out, string_view(buffer.data(), buffer.size()));
+}
 
 template <typename Char, typename UInt,
           FMT_ENABLE_IF(std::is_integral<UInt>::value)>
@@ -1631,14 +1742,20 @@ inline auto write_significand(Char* out, UInt significand, int significand_size,
                               int integral_size, Char decimal_point) -> Char* {
   if (!decimal_point)
     return format_decimal(out, significand, significand_size).end;
-  auto end = format_decimal(out + 1, significand, significand_size).end;
-  if (integral_size == 1) {
-    out[0] = out[1];
-  } else {
-    std::uninitialized_copy_n(out + 1, integral_size,
-                              make_checked(out, to_unsigned(integral_size)));
+  out += significand_size + 1;
+  Char* end = out;
+  int floating_size = significand_size - integral_size;
+  for (int i = floating_size / 2; i > 0; --i) {
+    out -= 2;
+    copy2(out, digits2(significand % 100));
+    significand /= 100;
   }
-  out[integral_size] = decimal_point;
+  if (floating_size % 2 != 0) {
+    *--out = static_cast<Char>('0' + significand % 10);
+    significand /= 10;
+  }
+  *--out = decimal_point;
+  format_decimal(out - integral_size, significand, integral_size);
   return end;
 }
 
@@ -1655,9 +1772,9 @@ inline auto write_significand(OutputIt out, UInt significand,
 }
 
 template <typename OutputIt, typename Char>
-inline auto write_significand(OutputIt out, const char* significand,
-                              int significand_size, int integral_size,
-                              Char decimal_point) -> OutputIt {
+FMT_CONSTEXPR auto write_significand(OutputIt out, const char* significand,
+                                     int significand_size, int integral_size,
+                                     Char decimal_point) -> OutputIt {
   out = detail::copy_str_noinline<Char>(significand,
                                         significand + integral_size, out);
   if (!decimal_point) return out;
@@ -1666,17 +1783,40 @@ inline auto write_significand(OutputIt out, const char* significand,
                                          significand + significand_size, out);
 }
 
-template <typename OutputIt, typename DecimalFP, typename Char>
-auto write_float(OutputIt out, const DecimalFP& fp,
-                 const basic_format_specs<Char>& specs, float_specs fspecs,
-                 Char decimal_point) -> OutputIt {
+template <typename OutputIt, typename Char, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+                                       int significand_size, int integral_size,
+                                       Char decimal_point,
+                                       const Grouping& grouping) -> OutputIt {
+  if (!grouping.separator()) {
+    return write_significand(out, significand, significand_size, integral_size,
+                             decimal_point);
+  }
+  auto buffer = basic_memory_buffer<Char>();
+  write_significand(buffer_appender<Char>(buffer), significand,
+                    significand_size, integral_size, decimal_point);
+  grouping.apply(
+      out, basic_string_view<Char>(buffer.data(), to_unsigned(integral_size)));
+  return detail::copy_str_noinline<Char>(buffer.data() + integral_size,
+                                         buffer.end(), out);
+}
+
+template <typename OutputIt, typename DecimalFP, typename Char,
+          typename Grouping = digit_grouping<Char>>
+FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& fp,
+                                    const basic_format_specs<Char>& specs,
+                                    float_specs fspecs, locale_ref loc)
+    -> OutputIt {
   auto significand = fp.significand;
   int significand_size = get_significand_size(fp);
-  static const Char zero = static_cast<Char>('0');
+  constexpr Char zero = static_cast<Char>('0');
   auto sign = fspecs.sign;
   size_t size = to_unsigned(significand_size) + (sign ? 1 : 0);
   using iterator = reserve_iterator<OutputIt>;
 
+  Char decimal_point =
+      fspecs.locale ? detail::decimal_point<Char>(loc) : static_cast<Char>('.');
+
   int output_exp = fp.exponent + significand_size - 1;
   auto use_exp_format = [=]() {
     if (fspecs.format == float_format::exp) return true;
@@ -1703,7 +1843,7 @@ auto write_float(OutputIt out, const DecimalFP& fp,
     size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits);
     char exp_char = fspecs.upper ? 'E' : 'e';
     auto write = [=](iterator it) {
-      if (sign) *it++ = static_cast<Char>(data::signs[sign]);
+      if (sign) *it++ = detail::sign<Char>(sign);
       // Insert a decimal point after the first digit and add an exponent.
       it = write_significand(it, significand, significand_size, 1,
                              decimal_point);
@@ -1728,10 +1868,12 @@ auto write_float(OutputIt out, const DecimalFP& fp,
       if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 1;
       if (num_zeros > 0) size += to_unsigned(num_zeros) + 1;
     }
+    auto grouping = Grouping(loc, fspecs.locale);
+    size += to_unsigned(grouping.count_separators(significand_size));
     return write_padded<align::right>(out, specs, size, [&](iterator it) {
-      if (sign) *it++ = static_cast<Char>(data::signs[sign]);
-      it = write_significand<Char>(it, significand, significand_size);
-      it = detail::fill_n(it, fp.exponent, zero);
+      if (sign) *it++ = detail::sign<Char>(sign);
+      it = write_significand<Char>(it, significand, significand_size,
+                                   fp.exponent, grouping);
       if (!fspecs.showpoint) return it;
       *it++ = decimal_point;
       return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
@@ -1740,10 +1882,12 @@ auto write_float(OutputIt out, const DecimalFP& fp,
     // 1234e-2 -> 12.34[0+]
     int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0;
     size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0);
+    auto grouping = Grouping(loc, fspecs.locale);
+    size += to_unsigned(grouping.count_separators(significand_size));
     return write_padded<align::right>(out, specs, size, [&](iterator it) {
-      if (sign) *it++ = static_cast<Char>(data::signs[sign]);
+      if (sign) *it++ = detail::sign<Char>(sign);
       it = write_significand(it, significand, significand_size, exp,
-                             decimal_point);
+                             decimal_point, grouping);
       return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
     });
   }
@@ -1756,7 +1900,7 @@ auto write_float(OutputIt out, const DecimalFP& fp,
   bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint;
   size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros);
   return write_padded<align::right>(out, specs, size, [&](iterator it) {
-    if (sign) *it++ = static_cast<Char>(data::signs[sign]);
+    if (sign) *it++ = detail::sign<Char>(sign);
     *it++ = zero;
     if (!pointy) return it;
     *it++ = decimal_point;
@@ -1765,26 +1909,97 @@ auto write_float(OutputIt out, const DecimalFP& fp,
   });
 }
 
+template <typename Char> class fallback_digit_grouping {
+ public:
+  constexpr fallback_digit_grouping(locale_ref, bool) {}
+
+  constexpr Char separator() const { return Char(); }
+
+  constexpr int count_separators(int) const { return 0; }
+
+  template <typename Out, typename C>
+  constexpr Out apply(Out out, basic_string_view<C>) const {
+    return out;
+  }
+};
+
+template <typename OutputIt, typename DecimalFP, typename Char>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& fp,
+                                 const basic_format_specs<Char>& specs,
+                                 float_specs fspecs, locale_ref loc)
+    -> OutputIt {
+  if (is_constant_evaluated()) {
+    return do_write_float<OutputIt, DecimalFP, Char,
+                          fallback_digit_grouping<Char>>(out, fp, specs, fspecs,
+                                                         loc);
+  } else {
+    return do_write_float(out, fp, specs, fspecs, loc);
+  }
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+FMT_CONSTEXPR20 bool isinf(T value) {
+  if (is_constant_evaluated()) {
+#if defined(__cpp_if_constexpr)
+    if constexpr (std::numeric_limits<double>::is_iec559) {
+      auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+      constexpr auto significand_bits =
+          dragonbox::float_info<double>::significand_bits;
+      return (bits & exponent_mask<double>()) &&
+             !(bits & ((uint64_t(1) << significand_bits) - 1));
+    }
+#endif
+  }
+  return std::isinf(value);
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+FMT_CONSTEXPR20 bool isfinite(T value) {
+  if (is_constant_evaluated()) {
+#if defined(__cpp_if_constexpr)
+    if constexpr (std::numeric_limits<double>::is_iec559) {
+      auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+      return (bits & exponent_mask<double>()) != exponent_mask<double>();
+    }
+#endif
+  }
+  return std::isfinite(value);
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+FMT_INLINE FMT_CONSTEXPR bool signbit(T value) {
+  if (is_constant_evaluated()) {
+#ifdef __cpp_if_constexpr
+    if constexpr (std::numeric_limits<double>::is_iec559) {
+      auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+      return (bits & (uint64_t(1) << (num_bits<uint64_t>() - 1))) != 0;
+    }
+#endif
+  }
+  return std::signbit(value);
+}
+
 template <typename Char, typename OutputIt, typename T,
           FMT_ENABLE_IF(std::is_floating_point<T>::value)>
-auto write(OutputIt out, T value, basic_format_specs<Char> specs,
-           locale_ref loc = {}) -> OutputIt {
+FMT_CONSTEXPR20 auto write(OutputIt out, T value,
+                           basic_format_specs<Char> specs, locale_ref loc = {})
+    -> OutputIt {
   if (const_check(!is_supported_floating_point(value))) return out;
   float_specs fspecs = parse_float_type_spec(specs);
   fspecs.sign = specs.sign;
-  if (std::signbit(value)) {  // value < 0 is false for NaN so use signbit.
+  if (detail::signbit(value)) {  // value < 0 is false for NaN so use signbit.
     fspecs.sign = sign::minus;
     value = -value;
   } else if (fspecs.sign == sign::minus) {
     fspecs.sign = sign::none;
   }
 
-  if (!std::isfinite(value))
-    return write_nonfinite(out, std::isinf(value), specs, fspecs);
+  if (!detail::isfinite(value))
+    return write_nonfinite(out, detail::isinf(value), specs, fspecs);
 
   if (specs.align == align::numeric && fspecs.sign) {
     auto it = reserve(out, 1);
-    *it++ = static_cast<Char>(data::signs[fspecs.sign]);
+    *it++ = detail::sign<Char>(fspecs.sign);
     out = base_iterator(out, it);
     fspecs.sign = sign::none;
     if (specs.width != 0) --specs.width;
@@ -1792,31 +2007,35 @@ auto write(OutputIt out, T value, basic_format_specs<Char> specs,
 
   memory_buffer buffer;
   if (fspecs.format == float_format::hex) {
-    if (fspecs.sign) buffer.push_back(data::signs[fspecs.sign]);
+    if (fspecs.sign) buffer.push_back(detail::sign<char>(fspecs.sign));
     snprintf_float(promote_float(value), specs.precision, fspecs, buffer);
     return write_bytes<align::right>(out, {buffer.data(), buffer.size()},
                                      specs);
   }
-  int precision = specs.precision >= 0 || !specs.type ? specs.precision : 6;
+  int precision = specs.precision >= 0 || specs.type == presentation_type::none
+                      ? specs.precision
+                      : 6;
   if (fspecs.format == float_format::exp) {
     if (precision == max_value<int>())
-      FMT_THROW(format_error("number is too big"));
+      throw_format_error("number is too big");
     else
       ++precision;
   }
   if (const_check(std::is_same<T, float>())) fspecs.binary32 = true;
-  fspecs.use_grisu = is_fast_float<T>();
+  if (!is_fast_float<T>()) fspecs.fallback = true;
   int exp = format_float(promote_float(value), precision, fspecs, buffer);
   fspecs.precision = precision;
-  Char point =
-      fspecs.locale ? decimal_point<Char>(loc) : static_cast<Char>('.');
   auto fp = big_decimal_fp{buffer.data(), static_cast<int>(buffer.size()), exp};
-  return write_float(out, fp, specs, fspecs, point);
+  return write_float(out, fp, specs, fspecs, loc);
 }
 
 template <typename Char, typename OutputIt, typename T,
           FMT_ENABLE_IF(is_fast_float<T>::value)>
-auto write(OutputIt out, T value) -> OutputIt {
+FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt {
+  if (is_constant_evaluated()) {
+    return write(out, value, basic_format_specs<Char>());
+  }
+
   if (const_check(!is_supported_floating_point(value))) return out;
 
   using floaty = conditional_t<std::is_same<T, long double>::value, double, T>;
@@ -1824,19 +2043,18 @@ auto write(OutputIt out, T value) -> OutputIt {
   auto bits = bit_cast<uint>(value);
 
   auto fspecs = float_specs();
-  auto sign_bit = bits & (uint(1) << (num_bits<uint>() - 1));
-  if (sign_bit != 0) {
+  if (detail::signbit(value)) {
     fspecs.sign = sign::minus;
     value = -value;
   }
 
-  static const auto specs = basic_format_specs<Char>();
+  constexpr auto specs = basic_format_specs<Char>();
   uint mask = exponent_mask<floaty>();
   if ((bits & mask) == mask)
     return write_nonfinite(out, std::isinf(value), specs, fspecs);
 
   auto dec = dragonbox::to_decimal(static_cast<floaty>(value));
-  return write_float(out, dec, specs, fspecs, static_cast<Char>('.'));
+  return write_float(out, dec, specs, fspecs, {});
 }
 
 template <typename Char, typename OutputIt, typename T,
@@ -1889,7 +2107,7 @@ FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
   return base_iterator(out, it);
 }
 
-// FMT_ENABLE_IF() condition separated to workaround MSVC bug
+// FMT_ENABLE_IF() condition separated to workaround an MSVC bug.
 template <
     typename Char, typename OutputIt, typename T,
     bool check =
@@ -1907,7 +2125,8 @@ template <typename Char, typename OutputIt, typename T,
 FMT_CONSTEXPR auto write(OutputIt out, T value,
                          const basic_format_specs<Char>& specs = {},
                          locale_ref = {}) -> OutputIt {
-  return specs.type && specs.type != 's'
+  return specs.type != presentation_type::none &&
+                 specs.type != presentation_type::string
              ? write(out, value ? 1 : 0, specs, {})
              : write_bytes(out, value ? "true" : "false", specs);
 }
@@ -1923,10 +2142,9 @@ template <typename Char, typename OutputIt>
 FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value)
     -> OutputIt {
   if (!value) {
-    FMT_THROW(format_error("string pointer is null"));
+    throw_format_error("string pointer is null");
   } else {
-    auto length = std::char_traits<Char>::length(value);
-    out = write(out, basic_string_view<Char>(value, length));
+    out = write(out, basic_string_view<Char>(value));
   }
   return out;
 }
@@ -1940,18 +2158,28 @@ auto write(OutputIt out, const T* value,
   return write_ptr<Char>(out, to_uintptr(value), &specs);
 }
 
-template <typename Char, typename OutputIt, typename T>
-FMT_CONSTEXPR auto write(OutputIt out, const T& value) ->
-    typename std::enable_if<
-        mapped_type_constant<T, basic_format_context<OutputIt, Char>>::value ==
-            type::custom_type,
-        OutputIt>::type {
-  using context_type = basic_format_context<OutputIt, Char>;
+// A write overload that handles implicit conversions.
+template <typename Char, typename OutputIt, typename T,
+          typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> enable_if_t<
+    std::is_class<T>::value && !is_string<T>::value &&
+        !std::is_same<T, Char>::value &&
+        !std::is_same<const T&,
+                      decltype(arg_mapper<Context>().map(value))>::value,
+    OutputIt> {
+  return write<Char>(out, arg_mapper<Context>().map(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+          typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value)
+    -> enable_if_t<mapped_type_constant<T, Context>::value == type::custom_type,
+                   OutputIt> {
   using formatter_type =
-      conditional_t<has_formatter<T, context_type>::value,
-                    typename context_type::template formatter_type<T>,
+      conditional_t<has_formatter<T, Context>::value,
+                    typename Context::template formatter_type<T>,
                     fallback_formatter<T, Char>>;
-  context_type ctx(out, {}, {});
+  auto ctx = Context(out, {}, {});
   return formatter_type().format(value, ctx);
 }
 
@@ -2368,6 +2596,7 @@ FMT_FORMAT_AS(unsigned long, unsigned long long);
 FMT_FORMAT_AS(Char*, const Char*);
 FMT_FORMAT_AS(std::basic_string<Char>, basic_string_view<Char>);
 FMT_FORMAT_AS(std::nullptr_t, const void*);
+FMT_FORMAT_AS(detail::byte, unsigned char);
 FMT_FORMAT_AS(detail::std_string_view<Char>, basic_string_view<Char>);
 
 template <typename Char>
@@ -2493,6 +2722,52 @@ template <> struct formatter<bytes> {
   }
 };
 
+// group_digits_view is not derived from view because it copies the argument.
+template <typename T> struct group_digits_view { T value; };
+
+/**
+  \rst
+  Returns a view that formats an integer value using ',' as a locale-independent
+  thousands separator.
+
+  **Example**::
+
+    fmt::print("{}", fmt::group_digits(12345));
+    // Output: "12,345"
+  \endrst
+ */
+template <typename T> auto group_digits(T value) -> group_digits_view<T> {
+  return {value};
+}
+
+template <typename T> struct formatter<group_digits_view<T>> : formatter<T> {
+ private:
+  detail::dynamic_format_specs<char> specs_;
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    using handler_type = detail::dynamic_specs_handler<ParseContext>;
+    detail::specs_checker<handler_type> handler(handler_type(specs_, ctx),
+                                                detail::type::int_type);
+    auto it = parse_format_specs(ctx.begin(), ctx.end(), handler);
+    detail::check_string_type_spec(specs_.type, ctx.error_handler());
+    return it;
+  }
+
+  template <typename FormatContext>
+  auto format(group_digits_view<T> t, FormatContext& ctx)
+      -> decltype(ctx.out()) {
+    detail::handle_dynamic_spec<detail::width_checker>(specs_.width,
+                                                       specs_.width_ref, ctx);
+    detail::handle_dynamic_spec<detail::precision_checker>(
+        specs_.precision, specs_.precision_ref, ctx);
+    return detail::write_int_localized(
+        ctx.out(), static_cast<detail::uint64_or_128_t<T>>(t.value), 0, specs_,
+        detail::digit_grouping<char>({"\3", ','}));
+  }
+};
+
 template <typename It, typename Sentinel, typename Char = char>
 struct join_view : detail::view {
   It begin;
@@ -2509,7 +2784,12 @@ using arg_join FMT_DEPRECATED_ALIAS = join_view<It, Sentinel, Char>;
 template <typename It, typename Sentinel, typename Char>
 struct formatter<join_view<It, Sentinel, Char>, Char> {
  private:
-  using value_type = typename std::iterator_traits<It>::value_type;
+  using value_type =
+#ifdef __cpp_lib_ranges
+      std::iter_value_t<It>;
+#else
+      typename std::iterator_traits<It>::value_type;
+#endif
   using context = buffer_context<Char>;
   using mapper = detail::arg_mapper<context>;
 
@@ -2543,11 +2823,13 @@ struct formatter<join_view<It, Sentinel, Char>, Char> {
     auto it = value.begin;
     auto out = ctx.out();
     if (it != value.end) {
-      out = value_formatter_.format(map(*it++), ctx);
+      out = value_formatter_.format(map(*it), ctx);
+      ++it;
       while (it != value.end) {
         out = detail::copy_str<Char>(value.sep.begin(), value.sep.end(), out);
         ctx.advance_to(out);
-        out = value_formatter_.format(map(*it++), ctx);
+        out = value_formatter_.format(map(*it), ctx);
+        ++it;
       }
     }
     return out;
@@ -2555,8 +2837,8 @@ struct formatter<join_view<It, Sentinel, Char>, Char> {
 };
 
 /**
-  Returns an object that formats the iterator range `[begin, end)` with
-  elements separated by `sep`.
+  Returns a view that formats the iterator range `[begin, end)` with elements
+  separated by `sep`.
  */
 template <typename It, typename Sentinel>
 auto join(It begin, Sentinel end, string_view sep) -> join_view<It, Sentinel> {
@@ -2565,7 +2847,7 @@ auto join(It begin, Sentinel end, string_view sep) -> join_view<It, Sentinel> {
 
 /**
   \rst
-  Returns an object that formats `range` with elements separated by `sep`.
+  Returns a view that formats `range` with elements separated by `sep`.
 
   **Example**::
 
@@ -2604,7 +2886,7 @@ inline auto to_string(const T& value) -> std::string {
 }
 
 template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
-inline auto to_string(T value) -> std::string {
+FMT_NODISCARD inline auto to_string(T value) -> std::string {
   // The buffer should be large enough to store the number including the sign
   // or "false" for bool.
   constexpr int max_size = detail::digits10<T>() + 2;
@@ -2614,7 +2896,7 @@ inline auto to_string(T value) -> std::string {
 }
 
 template <typename Char, size_t SIZE>
-auto to_string(const basic_memory_buffer<Char, SIZE>& buf)
+FMT_NODISCARD auto to_string(const basic_memory_buffer<Char, SIZE>& buf)
     -> std::basic_string<Char> {
   auto size = buf.size();
   detail::assume(size < std::basic_string<Char>().max_size());
@@ -2756,17 +3038,9 @@ constexpr auto operator"" _a(const char* s, size_t) -> detail::udl_arg<char> {
 }
 #  endif
 
-/**
-  \rst
-  User-defined literal equivalent of :func:`fmt::format`.
-
-  **Example**::
-
-    using namespace fmt::literals;
-    std::string message = "The answer is {}"_format(42);
-  \endrst
- */
-constexpr auto operator"" _format(const char* s, size_t n)
+// DEPRECATED!
+// User-defined literal equivalent of fmt::format.
+FMT_DEPRECATED constexpr auto operator"" _format(const char* s, size_t n)
     -> detail::udl_formatter<char> {
   return {{s, n}};
 }
diff --git a/contrib/fmt-8.0.1/include/fmt/locale.h b/contrib/fmt-8.1.1/include/fmt/locale.h
similarity index 100%
rename from contrib/fmt-8.0.1/include/fmt/locale.h
rename to contrib/fmt-8.1.1/include/fmt/locale.h
diff --git a/contrib/fmt-8.0.1/include/fmt/os.h b/contrib/fmt-8.1.1/include/fmt/os.h
similarity index 93%
rename from contrib/fmt-8.0.1/include/fmt/os.h
rename to contrib/fmt-8.1.1/include/fmt/os.h
index f6c0f32985..b64f8bbfa5 100644
--- a/contrib/fmt-8.0.1/include/fmt/os.h
+++ b/contrib/fmt-8.1.1/include/fmt/os.h
@@ -21,17 +21,20 @@
 
 #include "format.h"
 
+#ifndef FMT_USE_FCNTL
 // UWP doesn't provide _pipe.
-#if FMT_HAS_INCLUDE("winapifamily.h")
-#  include <winapifamily.h>
-#endif
-#if (FMT_HAS_INCLUDE(<fcntl.h>) || defined(__APPLE__) || \
-     defined(__linux__)) &&                              \
-    (!defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
-#  include <fcntl.h>  // for O_RDONLY
-#  define FMT_USE_FCNTL 1
-#else
-#  define FMT_USE_FCNTL 0
+#  if FMT_HAS_INCLUDE("winapifamily.h")
+#    include <winapifamily.h>
+#  endif
+#  if (FMT_HAS_INCLUDE(<fcntl.h>) || defined(__APPLE__) || \
+       defined(__linux__)) &&                              \
+      (!defined(WINAPI_FAMILY) ||                          \
+       (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+#    include <fcntl.h>  // for O_RDONLY
+#    define FMT_USE_FCNTL 1
+#  else
+#    define FMT_USE_FCNTL 0
+#  endif
 #endif
 
 #ifndef FMT_POSIX
@@ -390,23 +393,26 @@ struct ostream_params {
       : ostream_params(params...) {
     this->buffer_size = bs.value;
   }
+
+// Intel has a bug that results in failure to deduce a constructor
+// for empty parameter packs.
+#  if defined(__INTEL_COMPILER) && __INTEL_COMPILER < 2000
+  ostream_params(int new_oflag) : oflag(new_oflag) {}
+  ostream_params(detail::buffer_size bs) : buffer_size(bs.value) {}
+#  endif
 };
 
 FMT_END_DETAIL_NAMESPACE
 
-constexpr detail::buffer_size buffer_size;
+// Added {} below to work around default constructor error known to
+// occur in Xcode versions 7.2.1 and 8.2.1.
+constexpr detail::buffer_size buffer_size{};
 
 /** A fast output stream which is not thread-safe. */
 class FMT_API ostream final : private detail::buffer<char> {
  private:
   file file_;
 
-  void flush() {
-    if (size() == 0) return;
-    file_.write(data(), size());
-    clear();
-  }
-
   void grow(size_t) override;
 
   ostream(cstring_view path, const detail::ostream_params& params)
@@ -426,6 +432,12 @@ class FMT_API ostream final : private detail::buffer<char> {
     delete[] data();
   }
 
+  void flush() {
+    if (size() == 0) return;
+    file_.write(data(), size());
+    clear();
+  }
+
   template <typename... T>
   friend ostream output_file(cstring_view path, T... params);
 
@@ -500,7 +512,7 @@ class locale {
 
   // Converts string to floating-point number and advances str past the end
   // of the parsed input.
-  double strtod(const char*& str) const {
+  FMT_DEPRECATED double strtod(const char*& str) const {
     char* end = nullptr;
     double result = strtod_l(str, &end, locale_);
     str = end;
diff --git a/contrib/fmt-8.1.1/include/fmt/ostream.h b/contrib/fmt-8.1.1/include/fmt/ostream.h
new file mode 100644
index 0000000000..3d716ece84
--- /dev/null
+++ b/contrib/fmt-8.1.1/include/fmt/ostream.h
@@ -0,0 +1,135 @@
+// Formatting library for C++ - std::ostream support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OSTREAM_H_
+#define FMT_OSTREAM_H_
+
+#include <ostream>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+template <typename OutputIt, typename Char> class basic_printf_context;
+
+namespace detail {
+
+// Checks if T has a user-defined operator<<.
+template <typename T, typename Char, typename Enable = void>
+class is_streamable {
+ private:
+  template <typename U>
+  static auto test(int)
+      -> bool_constant<sizeof(std::declval<std::basic_ostream<Char>&>()
+                              << std::declval<U>()) != 0>;
+
+  template <typename> static auto test(...) -> std::false_type;
+
+  using result = decltype(test<T>(0));
+
+ public:
+  is_streamable() = default;
+
+  static const bool value = result::value;
+};
+
+// Formatting of built-in types and arrays is intentionally disabled because
+// it's handled by standard (non-ostream) formatters.
+template <typename T, typename Char>
+struct is_streamable<
+    T, Char,
+    enable_if_t<
+        std::is_arithmetic<T>::value || std::is_array<T>::value ||
+        std::is_pointer<T>::value || std::is_same<T, char8_type>::value ||
+        std::is_same<T, std::basic_string<Char>>::value ||
+        std::is_same<T, std_string_view<Char>>::value ||
+        (std::is_convertible<T, int>::value && !std::is_enum<T>::value)>>
+    : std::false_type {};
+
+// Write the content of buf to os.
+// It is a separate function rather than a part of vprint to simplify testing.
+template <typename Char>
+void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
+  const Char* buf_data = buf.data();
+  using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
+  unsigned_streamsize size = buf.size();
+  unsigned_streamsize max_size = to_unsigned(max_value<std::streamsize>());
+  do {
+    unsigned_streamsize n = size <= max_size ? size : max_size;
+    os.write(buf_data, static_cast<std::streamsize>(n));
+    buf_data += n;
+    size -= n;
+  } while (size != 0);
+}
+
+template <typename Char, typename T>
+void format_value(buffer<Char>& buf, const T& value,
+                  locale_ref loc = locale_ref()) {
+  auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+  auto&& output = std::basic_ostream<Char>(&format_buf);
+#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+  if (loc) output.imbue(loc.get<std::locale>());
+#endif
+  output << value;
+  output.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+  buf.try_resize(buf.size());
+}
+
+// Formats an object of type T that has an overloaded ostream operator<<.
+template <typename T, typename Char>
+struct fallback_formatter<T, Char, enable_if_t<is_streamable<T, Char>::value>>
+    : private formatter<basic_string_view<Char>, Char> {
+  using formatter<basic_string_view<Char>, Char>::parse;
+
+  template <typename OutputIt>
+  auto format(const T& value, basic_format_context<OutputIt, Char>& ctx)
+      -> OutputIt {
+    auto buffer = basic_memory_buffer<Char>();
+    format_value(buffer, value, ctx.locale());
+    return formatter<basic_string_view<Char>, Char>::format(
+        {buffer.data(), buffer.size()}, ctx);
+  }
+
+  // DEPRECATED!
+  template <typename OutputIt>
+  auto format(const T& value, basic_printf_context<OutputIt, Char>& ctx)
+      -> OutputIt {
+    auto buffer = basic_memory_buffer<Char>();
+    format_value(buffer, value, ctx.locale());
+    return std::copy(buffer.begin(), buffer.end(), ctx.out());
+  }
+};
+}  // namespace detail
+
+FMT_MODULE_EXPORT
+template <typename Char>
+void vprint(std::basic_ostream<Char>& os, basic_string_view<Char> format_str,
+            basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+  auto buffer = basic_memory_buffer<Char>();
+  detail::vformat_to(buffer, format_str, args);
+  detail::write_buffer(os, buffer);
+}
+
+/**
+  \rst
+  Prints formatted data to the stream *os*.
+
+  **Example**::
+
+    fmt::print(cerr, "Don't {}!", "panic");
+  \endrst
+ */
+FMT_MODULE_EXPORT
+template <typename S, typename... Args,
+          typename Char = enable_if_t<detail::is_string<S>::value, char_t<S>>>
+void print(std::basic_ostream<Char>& os, const S& format_str, Args&&... args) {
+  vprint(os, to_string_view(format_str),
+         fmt::make_args_checked<Args...>(format_str, args...));
+}
+FMT_END_NAMESPACE
+
+#endif  // FMT_OSTREAM_H_
diff --git a/contrib/fmt-8.0.1/include/fmt/printf.h b/contrib/fmt-8.1.1/include/fmt/printf.h
similarity index 97%
rename from contrib/fmt-8.0.1/include/fmt/printf.h
rename to contrib/fmt-8.1.1/include/fmt/printf.h
index 3a3cd15283..19d550f6cf 100644
--- a/contrib/fmt-8.0.1/include/fmt/printf.h
+++ b/contrib/fmt-8.1.1/include/fmt/printf.h
@@ -233,7 +233,7 @@ class printf_arg_formatter : public arg_formatter<Char> {
 
   OutputIt write_null_pointer(bool is_string = false) {
     auto s = this->specs;
-    s.type = 0;
+    s.type = presentation_type::none;
     return write_bytes(this->out, is_string ? "(null)" : "(nil)", s);
   }
 
@@ -249,8 +249,10 @@ class printf_arg_formatter : public arg_formatter<Char> {
     // std::is_same instead.
     if (std::is_same<T, Char>::value) {
       format_specs fmt_specs = this->specs;
-      if (fmt_specs.type && fmt_specs.type != 'c')
+      if (fmt_specs.type != presentation_type::none &&
+          fmt_specs.type != presentation_type::chr) {
         return (*this)(static_cast<int>(value));
+      }
       fmt_specs.sign = sign::none;
       fmt_specs.alt = false;
       fmt_specs.fill[0] = ' ';  // Ignore '0' flag for char types.
@@ -271,13 +273,13 @@ class printf_arg_formatter : public arg_formatter<Char> {
   /** Formats a null-terminated C string. */
   OutputIt operator()(const char* value) {
     if (value) return base::operator()(value);
-    return write_null_pointer(this->specs.type != 'p');
+    return write_null_pointer(this->specs.type != presentation_type::pointer);
   }
 
   /** Formats a null-terminated wide C string. */
   OutputIt operator()(const wchar_t* value) {
     if (value) return base::operator()(value);
-    return write_null_pointer(this->specs.type != 'p');
+    return write_null_pointer(this->specs.type != presentation_type::pointer);
   }
 
   OutputIt operator()(basic_string_view<Char> value) {
@@ -490,13 +492,13 @@ void vprintf(buffer<Char>& buf, basic_string_view<Char> format,
 
     // Parse type.
     if (it == end) FMT_THROW(format_error("invalid format string"));
-    specs.type = static_cast<char>(*it++);
+    char type = static_cast<char>(*it++);
     if (arg.is_integral()) {
       // Normalize type.
-      switch (specs.type) {
+      switch (type) {
       case 'i':
       case 'u':
-        specs.type = 'd';
+        type = 'd';
         break;
       case 'c':
         visit_format_arg(
@@ -505,6 +507,9 @@ void vprintf(buffer<Char>& buf, basic_string_view<Char> format,
         break;
       }
     }
+    specs.type = parse_presentation_type(type);
+    if (specs.type == presentation_type::none)
+      parse_ctx.on_error("invalid type specifier");
 
     start = it;
 
diff --git a/contrib/fmt-8.1.1/include/fmt/ranges.h b/contrib/fmt-8.1.1/include/fmt/ranges.h
new file mode 100644
index 0000000000..eb9fb8a92d
--- /dev/null
+++ b/contrib/fmt-8.1.1/include/fmt/ranges.h
@@ -0,0 +1,793 @@
+// Formatting library for C++ - experimental range support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+//
+// Copyright (c) 2018 - present, Remotion (Igor Schulz)
+// All Rights Reserved
+// {fmt} support for ranges, containers and types tuple interface.
+
+#ifndef FMT_RANGES_H_
+#define FMT_RANGES_H_
+
+#include <initializer_list>
+#include <tuple>
+#include <type_traits>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename RangeT, typename OutputIterator>
+OutputIterator copy(const RangeT& range, OutputIterator out) {
+  for (auto it = range.begin(), end = range.end(); it != end; ++it)
+    *out++ = *it;
+  return out;
+}
+
+template <typename OutputIterator>
+OutputIterator copy(const char* str, OutputIterator out) {
+  while (*str) *out++ = *str++;
+  return out;
+}
+
+template <typename OutputIterator>
+OutputIterator copy(char ch, OutputIterator out) {
+  *out++ = ch;
+  return out;
+}
+
+template <typename OutputIterator>
+OutputIterator copy(wchar_t ch, OutputIterator out) {
+  *out++ = ch;
+  return out;
+}
+
+// Returns true if T has a std::string-like interface, like std::string_view.
+template <typename T> class is_std_string_like {
+  template <typename U>
+  static auto check(U* p)
+      -> decltype((void)p->find('a'), p->length(), (void)p->data(), int());
+  template <typename> static void check(...);
+
+ public:
+  static FMT_CONSTEXPR_DECL const bool value =
+      is_string<T>::value ||
+      std::is_convertible<T, std_string_view<char>>::value ||
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Char>
+struct is_std_string_like<fmt::basic_string_view<Char>> : std::true_type {};
+
+template <typename T> class is_map {
+  template <typename U> static auto check(U*) -> typename U::mapped_type;
+  template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_MAP_AS_LIST
+  static FMT_CONSTEXPR_DECL const bool value = false;
+#else
+  static FMT_CONSTEXPR_DECL const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+#endif
+};
+
+template <typename T> class is_set {
+  template <typename U> static auto check(U*) -> typename U::key_type;
+  template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_SET_AS_LIST
+  static FMT_CONSTEXPR_DECL const bool value = false;
+#else
+  static FMT_CONSTEXPR_DECL const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value && !is_map<T>::value;
+#endif
+};
+
+template <typename... Ts> struct conditional_helper {};
+
+template <typename T, typename _ = void> struct is_range_ : std::false_type {};
+
+#if !FMT_MSC_VER || FMT_MSC_VER > 1800
+
+#  define FMT_DECLTYPE_RETURN(val)  \
+    ->decltype(val) { return val; } \
+    static_assert(                  \
+        true, "")  // This makes it so that a semicolon is required after the
+                   // macro, which helps clang-format handle the formatting.
+
+// C array overload
+template <typename T, std::size_t N>
+auto range_begin(const T (&arr)[N]) -> const T* {
+  return arr;
+}
+template <typename T, std::size_t N>
+auto range_end(const T (&arr)[N]) -> const T* {
+  return arr + N;
+}
+
+template <typename T, typename Enable = void>
+struct has_member_fn_begin_end_t : std::false_type {};
+
+template <typename T>
+struct has_member_fn_begin_end_t<T, void_t<decltype(std::declval<T>().begin()),
+                                           decltype(std::declval<T>().end())>>
+    : std::true_type {};
+
+// Member function overload
+template <typename T>
+auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
+template <typename T>
+auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
+
+// ADL overload. Only participates in overload resolution if member functions
+// are not found.
+template <typename T>
+auto range_begin(T&& rng)
+    -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+                   decltype(begin(static_cast<T&&>(rng)))> {
+  return begin(static_cast<T&&>(rng));
+}
+template <typename T>
+auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+                                       decltype(end(static_cast<T&&>(rng)))> {
+  return end(static_cast<T&&>(rng));
+}
+
+template <typename T, typename Enable = void>
+struct has_const_begin_end : std::false_type {};
+template <typename T, typename Enable = void>
+struct has_mutable_begin_end : std::false_type {};
+
+template <typename T>
+struct has_const_begin_end<
+    T,
+    void_t<
+        decltype(detail::range_begin(std::declval<const remove_cvref_t<T>&>())),
+        decltype(detail::range_end(std::declval<const remove_cvref_t<T>&>()))>>
+    : std::true_type {};
+
+template <typename T>
+struct has_mutable_begin_end<
+    T, void_t<decltype(detail::range_begin(std::declval<T>())),
+              decltype(detail::range_end(std::declval<T>())),
+              enable_if_t<std::is_copy_constructible<T>::value>>>
+    : std::true_type {};
+
+template <typename T>
+struct is_range_<T, void>
+    : std::integral_constant<bool, (has_const_begin_end<T>::value ||
+                                    has_mutable_begin_end<T>::value)> {};
+#  undef FMT_DECLTYPE_RETURN
+#endif
+
+// tuple_size and tuple_element check.
+template <typename T> class is_tuple_like_ {
+  template <typename U>
+  static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
+  template <typename> static void check(...);
+
+ public:
+  static FMT_CONSTEXPR_DECL const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+// Check for integer_sequence
+#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VER >= 1900
+template <typename T, T... N>
+using integer_sequence = std::integer_sequence<T, N...>;
+template <size_t... N> using index_sequence = std::index_sequence<N...>;
+template <size_t N> using make_index_sequence = std::make_index_sequence<N>;
+#else
+template <typename T, T... N> struct integer_sequence {
+  using value_type = T;
+
+  static FMT_CONSTEXPR size_t size() { return sizeof...(N); }
+};
+
+template <size_t... N> using index_sequence = integer_sequence<size_t, N...>;
+
+template <typename T, size_t N, T... Ns>
+struct make_integer_sequence : make_integer_sequence<T, N - 1, N - 1, Ns...> {};
+template <typename T, T... Ns>
+struct make_integer_sequence<T, 0, Ns...> : integer_sequence<T, Ns...> {};
+
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+#endif
+
+template <class Tuple, class F, size_t... Is>
+void for_each(index_sequence<Is...>, Tuple&& tup, F&& f) FMT_NOEXCEPT {
+  using std::get;
+  // using free function get<I>(T) now.
+  const int _[] = {0, ((void)f(get<Is>(tup)), 0)...};
+  (void)_;  // blocks warnings
+}
+
+template <class T>
+FMT_CONSTEXPR make_index_sequence<std::tuple_size<T>::value> get_indexes(
+    T const&) {
+  return {};
+}
+
+template <class Tuple, class F> void for_each(Tuple&& tup, F&& f) {
+  const auto indexes = get_indexes(tup);
+  for_each(indexes, std::forward<Tuple>(tup), std::forward<F>(f));
+}
+
+template <typename Range>
+using value_type =
+    remove_cvref_t<decltype(*detail::range_begin(std::declval<Range>()))>;
+
+template <typename OutputIt> OutputIt write_delimiter(OutputIt out) {
+  *out++ = ',';
+  *out++ = ' ';
+  return out;
+}
+
+struct singleton {
+  unsigned char upper;
+  unsigned char lower_count;
+};
+
+inline auto is_printable(uint16_t x, const singleton* singletons,
+                         size_t singletons_size,
+                         const unsigned char* singleton_lowers,
+                         const unsigned char* normal, size_t normal_size)
+    -> bool {
+  auto upper = x >> 8;
+  auto lower_start = 0;
+  for (size_t i = 0; i < singletons_size; ++i) {
+    auto s = singletons[i];
+    auto lower_end = lower_start + s.lower_count;
+    if (upper < s.upper) break;
+    if (upper == s.upper) {
+      for (auto j = lower_start; j < lower_end; ++j) {
+        if (singleton_lowers[j] == (x & 0xff)) return false;
+      }
+    }
+    lower_start = lower_end;
+  }
+
+  auto xsigned = static_cast<int>(x);
+  auto current = true;
+  for (size_t i = 0; i < normal_size; ++i) {
+    auto v = static_cast<int>(normal[i]);
+    auto len = (v & 0x80) != 0 ? (v & 0x7f) << 8 | normal[++i] : v;
+    xsigned -= len;
+    if (xsigned < 0) break;
+    current = !current;
+  }
+  return current;
+}
+
+// Returns true iff the code point cp is printable.
+// This code is generated by support/printable.py.
+inline auto is_printable(uint32_t cp) -> bool {
+  static constexpr singleton singletons0[] = {
+      {0x00, 1},  {0x03, 5},  {0x05, 6},  {0x06, 3},  {0x07, 6},  {0x08, 8},
+      {0x09, 17}, {0x0a, 28}, {0x0b, 25}, {0x0c, 20}, {0x0d, 16}, {0x0e, 13},
+      {0x0f, 4},  {0x10, 3},  {0x12, 18}, {0x13, 9},  {0x16, 1},  {0x17, 5},
+      {0x18, 2},  {0x19, 3},  {0x1a, 7},  {0x1c, 2},  {0x1d, 1},  {0x1f, 22},
+      {0x20, 3},  {0x2b, 3},  {0x2c, 2},  {0x2d, 11}, {0x2e, 1},  {0x30, 3},
+      {0x31, 2},  {0x32, 1},  {0xa7, 2},  {0xa9, 2},  {0xaa, 4},  {0xab, 8},
+      {0xfa, 2},  {0xfb, 5},  {0xfd, 4},  {0xfe, 3},  {0xff, 9},
+  };
+  static constexpr unsigned char singletons0_lower[] = {
+      0xad, 0x78, 0x79, 0x8b, 0x8d, 0xa2, 0x30, 0x57, 0x58, 0x8b, 0x8c, 0x90,
+      0x1c, 0x1d, 0xdd, 0x0e, 0x0f, 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f,
+      0x5c, 0x5d, 0x5f, 0xb5, 0xe2, 0x84, 0x8d, 0x8e, 0x91, 0x92, 0xa9, 0xb1,
+      0xba, 0xbb, 0xc5, 0xc6, 0xc9, 0xca, 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04,
+      0x11, 0x12, 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, 0x3d, 0x49, 0x4a, 0x5d,
+      0x84, 0x8e, 0x92, 0xa9, 0xb1, 0xb4, 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf,
+      0xe4, 0xe5, 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+      0x3b, 0x45, 0x46, 0x49, 0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d,
+      0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49, 0x57, 0x64, 0x65, 0x8d,
+      0x91, 0xa9, 0xb4, 0xba, 0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d,
+      0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5,
+      0xd7, 0xf0, 0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7,
+      0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49,
+      0x4e, 0x4f, 0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7,
+      0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7,
+      0xfe, 0xff, 0x80, 0x0d, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e,
+      0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf, 0xbb, 0xbc, 0xfa, 0x16,
+      0x17, 0x1e, 0x1f, 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e,
+      0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1, 0xf5, 0x72, 0x73, 0x8f,
+      0x74, 0x75, 0x96, 0x2f, 0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf,
+      0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, 0x30, 0x8f, 0x1f, 0xc0,
+      0xc1, 0xce, 0xff, 0x4e, 0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27,
+      0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, 0x42, 0x45, 0x90, 0x91,
+      0xfe, 0xff, 0x53, 0x67, 0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7,
+      0xfe, 0xff,
+  };
+  static constexpr singleton singletons1[] = {
+      {0x00, 6},  {0x01, 1}, {0x03, 1},  {0x04, 2}, {0x08, 8},  {0x09, 2},
+      {0x0a, 5},  {0x0b, 2}, {0x0e, 4},  {0x10, 1}, {0x11, 2},  {0x12, 5},
+      {0x13, 17}, {0x14, 1}, {0x15, 2},  {0x17, 2}, {0x19, 13}, {0x1c, 5},
+      {0x1d, 8},  {0x24, 1}, {0x6a, 3},  {0x6b, 2}, {0xbc, 2},  {0xd1, 2},
+      {0xd4, 12}, {0xd5, 9}, {0xd6, 2},  {0xd7, 2}, {0xda, 1},  {0xe0, 5},
+      {0xe1, 2},  {0xe8, 2}, {0xee, 32}, {0xf0, 4}, {0xf8, 2},  {0xf9, 2},
+      {0xfa, 2},  {0xfb, 1},
+  };
+  static constexpr unsigned char singletons1_lower[] = {
+      0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e, 0x9e, 0x9f, 0x06, 0x07,
+      0x09, 0x36, 0x3d, 0x3e, 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36,
+      0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd, 0x35, 0xe0, 0x12, 0x87,
+      0x89, 0x8e, 0x9e, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+      0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, 0x5c, 0xb6, 0xb7, 0x1b,
+      0x1c, 0x07, 0x08, 0x0a, 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9,
+      0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, 0x0a, 0x3b, 0x3e, 0x66,
+      0x69, 0x8f, 0x92, 0x6f, 0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27,
+      0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, 0xa8, 0xad, 0xba, 0xbc,
+      0xc4, 0x06, 0x0b, 0x0c, 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7,
+      0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, 0x3e, 0x3f, 0xc5, 0xc6,
+      0x04, 0x20, 0x23, 0x25, 0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c,
+      0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63, 0x65, 0x66,
+      0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0,
+      0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93,
+  };
+  static constexpr unsigned char normal0[] = {
+      0x00, 0x20, 0x5f, 0x22, 0x82, 0xdf, 0x04, 0x82, 0x44, 0x08, 0x1b, 0x04,
+      0x06, 0x11, 0x81, 0xac, 0x0e, 0x80, 0xab, 0x35, 0x28, 0x0b, 0x80, 0xe0,
+      0x03, 0x19, 0x08, 0x01, 0x04, 0x2f, 0x04, 0x34, 0x04, 0x07, 0x03, 0x01,
+      0x07, 0x06, 0x07, 0x11, 0x0a, 0x50, 0x0f, 0x12, 0x07, 0x55, 0x07, 0x03,
+      0x04, 0x1c, 0x0a, 0x09, 0x03, 0x08, 0x03, 0x07, 0x03, 0x02, 0x03, 0x03,
+      0x03, 0x0c, 0x04, 0x05, 0x03, 0x0b, 0x06, 0x01, 0x0e, 0x15, 0x05, 0x3a,
+      0x03, 0x11, 0x07, 0x06, 0x05, 0x10, 0x07, 0x57, 0x07, 0x02, 0x07, 0x15,
+      0x0d, 0x50, 0x04, 0x43, 0x03, 0x2d, 0x03, 0x01, 0x04, 0x11, 0x06, 0x0f,
+      0x0c, 0x3a, 0x04, 0x1d, 0x25, 0x5f, 0x20, 0x6d, 0x04, 0x6a, 0x25, 0x80,
+      0xc8, 0x05, 0x82, 0xb0, 0x03, 0x1a, 0x06, 0x82, 0xfd, 0x03, 0x59, 0x07,
+      0x15, 0x0b, 0x17, 0x09, 0x14, 0x0c, 0x14, 0x0c, 0x6a, 0x06, 0x0a, 0x06,
+      0x1a, 0x06, 0x59, 0x07, 0x2b, 0x05, 0x46, 0x0a, 0x2c, 0x04, 0x0c, 0x04,
+      0x01, 0x03, 0x31, 0x0b, 0x2c, 0x04, 0x1a, 0x06, 0x0b, 0x03, 0x80, 0xac,
+      0x06, 0x0a, 0x06, 0x21, 0x3f, 0x4c, 0x04, 0x2d, 0x03, 0x74, 0x08, 0x3c,
+      0x03, 0x0f, 0x03, 0x3c, 0x07, 0x38, 0x08, 0x2b, 0x05, 0x82, 0xff, 0x11,
+      0x18, 0x08, 0x2f, 0x11, 0x2d, 0x03, 0x20, 0x10, 0x21, 0x0f, 0x80, 0x8c,
+      0x04, 0x82, 0x97, 0x19, 0x0b, 0x15, 0x88, 0x94, 0x05, 0x2f, 0x05, 0x3b,
+      0x07, 0x02, 0x0e, 0x18, 0x09, 0x80, 0xb3, 0x2d, 0x74, 0x0c, 0x80, 0xd6,
+      0x1a, 0x0c, 0x05, 0x80, 0xff, 0x05, 0x80, 0xdf, 0x0c, 0xee, 0x0d, 0x03,
+      0x84, 0x8d, 0x03, 0x37, 0x09, 0x81, 0x5c, 0x14, 0x80, 0xb8, 0x08, 0x80,
+      0xcb, 0x2a, 0x38, 0x03, 0x0a, 0x06, 0x38, 0x08, 0x46, 0x08, 0x0c, 0x06,
+      0x74, 0x0b, 0x1e, 0x03, 0x5a, 0x04, 0x59, 0x09, 0x80, 0x83, 0x18, 0x1c,
+      0x0a, 0x16, 0x09, 0x4c, 0x04, 0x80, 0x8a, 0x06, 0xab, 0xa4, 0x0c, 0x17,
+      0x04, 0x31, 0xa1, 0x04, 0x81, 0xda, 0x26, 0x07, 0x0c, 0x05, 0x05, 0x80,
+      0xa5, 0x11, 0x81, 0x6d, 0x10, 0x78, 0x28, 0x2a, 0x06, 0x4c, 0x04, 0x80,
+      0x8d, 0x04, 0x80, 0xbe, 0x03, 0x1b, 0x03, 0x0f, 0x0d,
+  };
+  static constexpr unsigned char normal1[] = {
+      0x5e, 0x22, 0x7b, 0x05, 0x03, 0x04, 0x2d, 0x03, 0x66, 0x03, 0x01, 0x2f,
+      0x2e, 0x80, 0x82, 0x1d, 0x03, 0x31, 0x0f, 0x1c, 0x04, 0x24, 0x09, 0x1e,
+      0x05, 0x2b, 0x05, 0x44, 0x04, 0x0e, 0x2a, 0x80, 0xaa, 0x06, 0x24, 0x04,
+      0x24, 0x04, 0x28, 0x08, 0x34, 0x0b, 0x01, 0x80, 0x90, 0x81, 0x37, 0x09,
+      0x16, 0x0a, 0x08, 0x80, 0x98, 0x39, 0x03, 0x63, 0x08, 0x09, 0x30, 0x16,
+      0x05, 0x21, 0x03, 0x1b, 0x05, 0x01, 0x40, 0x38, 0x04, 0x4b, 0x05, 0x2f,
+      0x04, 0x0a, 0x07, 0x09, 0x07, 0x40, 0x20, 0x27, 0x04, 0x0c, 0x09, 0x36,
+      0x03, 0x3a, 0x05, 0x1a, 0x07, 0x04, 0x0c, 0x07, 0x50, 0x49, 0x37, 0x33,
+      0x0d, 0x33, 0x07, 0x2e, 0x08, 0x0a, 0x81, 0x26, 0x52, 0x4e, 0x28, 0x08,
+      0x2a, 0x56, 0x1c, 0x14, 0x17, 0x09, 0x4e, 0x04, 0x1e, 0x0f, 0x43, 0x0e,
+      0x19, 0x07, 0x0a, 0x06, 0x48, 0x08, 0x27, 0x09, 0x75, 0x0b, 0x3f, 0x41,
+      0x2a, 0x06, 0x3b, 0x05, 0x0a, 0x06, 0x51, 0x06, 0x01, 0x05, 0x10, 0x03,
+      0x05, 0x80, 0x8b, 0x62, 0x1e, 0x48, 0x08, 0x0a, 0x80, 0xa6, 0x5e, 0x22,
+      0x45, 0x0b, 0x0a, 0x06, 0x0d, 0x13, 0x39, 0x07, 0x0a, 0x36, 0x2c, 0x04,
+      0x10, 0x80, 0xc0, 0x3c, 0x64, 0x53, 0x0c, 0x48, 0x09, 0x0a, 0x46, 0x45,
+      0x1b, 0x48, 0x08, 0x53, 0x1d, 0x39, 0x81, 0x07, 0x46, 0x0a, 0x1d, 0x03,
+      0x47, 0x49, 0x37, 0x03, 0x0e, 0x08, 0x0a, 0x06, 0x39, 0x07, 0x0a, 0x81,
+      0x36, 0x19, 0x80, 0xb7, 0x01, 0x0f, 0x32, 0x0d, 0x83, 0x9b, 0x66, 0x75,
+      0x0b, 0x80, 0xc4, 0x8a, 0xbc, 0x84, 0x2f, 0x8f, 0xd1, 0x82, 0x47, 0xa1,
+      0xb9, 0x82, 0x39, 0x07, 0x2a, 0x04, 0x02, 0x60, 0x26, 0x0a, 0x46, 0x0a,
+      0x28, 0x05, 0x13, 0x82, 0xb0, 0x5b, 0x65, 0x4b, 0x04, 0x39, 0x07, 0x11,
+      0x40, 0x05, 0x0b, 0x02, 0x0e, 0x97, 0xf8, 0x08, 0x84, 0xd6, 0x2a, 0x09,
+      0xa2, 0xf7, 0x81, 0x1f, 0x31, 0x03, 0x11, 0x04, 0x08, 0x81, 0x8c, 0x89,
+      0x04, 0x6b, 0x05, 0x0d, 0x03, 0x09, 0x07, 0x10, 0x93, 0x60, 0x80, 0xf6,
+      0x0a, 0x73, 0x08, 0x6e, 0x17, 0x46, 0x80, 0x9a, 0x14, 0x0c, 0x57, 0x09,
+      0x19, 0x80, 0x87, 0x81, 0x47, 0x03, 0x85, 0x42, 0x0f, 0x15, 0x85, 0x50,
+      0x2b, 0x80, 0xd5, 0x2d, 0x03, 0x1a, 0x04, 0x02, 0x81, 0x70, 0x3a, 0x05,
+      0x01, 0x85, 0x00, 0x80, 0xd7, 0x29, 0x4c, 0x04, 0x0a, 0x04, 0x02, 0x83,
+      0x11, 0x44, 0x4c, 0x3d, 0x80, 0xc2, 0x3c, 0x06, 0x01, 0x04, 0x55, 0x05,
+      0x1b, 0x34, 0x02, 0x81, 0x0e, 0x2c, 0x04, 0x64, 0x0c, 0x56, 0x0a, 0x80,
+      0xae, 0x38, 0x1d, 0x0d, 0x2c, 0x04, 0x09, 0x07, 0x02, 0x0e, 0x06, 0x80,
+      0x9a, 0x83, 0xd8, 0x08, 0x0d, 0x03, 0x0d, 0x03, 0x74, 0x0c, 0x59, 0x07,
+      0x0c, 0x14, 0x0c, 0x04, 0x38, 0x08, 0x0a, 0x06, 0x28, 0x08, 0x22, 0x4e,
+      0x81, 0x54, 0x0c, 0x15, 0x03, 0x03, 0x05, 0x07, 0x09, 0x19, 0x07, 0x07,
+      0x09, 0x03, 0x0d, 0x07, 0x29, 0x80, 0xcb, 0x25, 0x0a, 0x84, 0x06,
+  };
+  auto lower = static_cast<uint16_t>(cp);
+  if (cp < 0x10000) {
+    return is_printable(lower, singletons0,
+                        sizeof(singletons0) / sizeof(*singletons0),
+                        singletons0_lower, normal0, sizeof(normal0));
+  }
+  if (cp < 0x20000) {
+    return is_printable(lower, singletons1,
+                        sizeof(singletons1) / sizeof(*singletons1),
+                        singletons1_lower, normal1, sizeof(normal1));
+  }
+  if (0x2a6de <= cp && cp < 0x2a700) return false;
+  if (0x2b735 <= cp && cp < 0x2b740) return false;
+  if (0x2b81e <= cp && cp < 0x2b820) return false;
+  if (0x2cea2 <= cp && cp < 0x2ceb0) return false;
+  if (0x2ebe1 <= cp && cp < 0x2f800) return false;
+  if (0x2fa1e <= cp && cp < 0x30000) return false;
+  if (0x3134b <= cp && cp < 0xe0100) return false;
+  if (0xe01f0 <= cp && cp < 0x110000) return false;
+  return cp < 0x110000;
+}
+
+inline auto needs_escape(uint32_t cp) -> bool {
+  return cp < 0x20 || cp == 0x7f || cp == '"' || cp == '\\' ||
+         !is_printable(cp);
+}
+
+template <typename Char> struct find_escape_result {
+  const Char* begin;
+  const Char* end;
+  uint32_t cp;
+};
+
+template <typename Char>
+auto find_escape(const Char* begin, const Char* end)
+    -> find_escape_result<Char> {
+  for (; begin != end; ++begin) {
+    auto cp = static_cast<typename std::make_unsigned<Char>::type>(*begin);
+    if (sizeof(Char) == 1 && cp >= 0x80) continue;
+    if (needs_escape(cp)) return {begin, begin + 1, cp};
+  }
+  return {begin, nullptr, 0};
+}
+
+inline auto find_escape(const char* begin, const char* end)
+    -> find_escape_result<char> {
+  if (!is_utf8()) return find_escape<char>(begin, end);
+  auto result = find_escape_result<char>{end, nullptr, 0};
+  for_each_codepoint(string_view(begin, to_unsigned(end - begin)),
+                     [&](uint32_t cp, string_view sv) {
+                       if (needs_escape(cp)) {
+                         result = {sv.begin(), sv.end(), cp};
+                         return false;
+                       }
+                       return true;
+                     });
+  return result;
+}
+
+template <typename Char, typename OutputIt>
+auto write_range_entry(OutputIt out, basic_string_view<Char> str) -> OutputIt {
+  *out++ = '"';
+  auto begin = str.begin(), end = str.end();
+  do {
+    auto escape = find_escape(begin, end);
+    out = copy_str<Char>(begin, escape.begin, out);
+    begin = escape.end;
+    if (!begin) break;
+    auto c = static_cast<Char>(escape.cp);
+    switch (escape.cp) {
+    case '\n':
+      *out++ = '\\';
+      c = 'n';
+      break;
+    case '\r':
+      *out++ = '\\';
+      c = 'r';
+      break;
+    case '\t':
+      *out++ = '\\';
+      c = 't';
+      break;
+    case '"':
+      FMT_FALLTHROUGH;
+    case '\\':
+      *out++ = '\\';
+      break;
+    default:
+      if (is_utf8()) {
+        if (escape.cp < 0x100) {
+          out = format_to(out, "\\x{:02x}", escape.cp);
+          continue;
+        }
+        if (escape.cp < 0x10000) {
+          out = format_to(out, "\\u{:04x}", escape.cp);
+          continue;
+        }
+        if (escape.cp < 0x110000) {
+          out = format_to(out, "\\U{:08x}", escape.cp);
+          continue;
+        }
+      }
+      for (Char escape_char : basic_string_view<Char>(
+               escape.begin, to_unsigned(escape.end - escape.begin))) {
+        out = format_to(
+            out, "\\x{:02x}",
+            static_cast<typename std::make_unsigned<Char>::type>(escape_char));
+      }
+      continue;
+    }
+    *out++ = c;
+  } while (begin != end);
+  *out++ = '"';
+  return out;
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(std::is_convertible<T, std_string_view<char>>::value)>
+inline auto write_range_entry(OutputIt out, const T& str) -> OutputIt {
+  auto sv = std_string_view<Char>(str);
+  return write_range_entry<Char>(out, basic_string_view<Char>(sv));
+}
+
+template <typename Char, typename OutputIt, typename Arg,
+          FMT_ENABLE_IF(std::is_same<Arg, Char>::value)>
+OutputIt write_range_entry(OutputIt out, const Arg v) {
+  *out++ = '\'';
+  *out++ = v;
+  *out++ = '\'';
+  return out;
+}
+
+template <
+    typename Char, typename OutputIt, typename Arg,
+    FMT_ENABLE_IF(!is_std_string_like<typename std::decay<Arg>::type>::value &&
+                  !std::is_same<Arg, Char>::value)>
+OutputIt write_range_entry(OutputIt out, const Arg& v) {
+  return write<Char>(out, v);
+}
+
+}  // namespace detail
+
+template <typename T> struct is_tuple_like {
+  static FMT_CONSTEXPR_DECL const bool value =
+      detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
+};
+
+template <typename TupleT, typename Char>
+struct formatter<TupleT, Char, enable_if_t<fmt::is_tuple_like<TupleT>::value>> {
+ private:
+  // C++11 generic lambda for format().
+  template <typename FormatContext> struct format_each {
+    template <typename T> void operator()(const T& v) {
+      if (i > 0) out = detail::write_delimiter(out);
+      out = detail::write_range_entry<Char>(out, v);
+      ++i;
+    }
+    int i;
+    typename FormatContext::iterator& out;
+  };
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext = format_context>
+  auto format(const TupleT& values, FormatContext& ctx) -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    *out++ = '(';
+    detail::for_each(values, format_each<FormatContext>{0, out});
+    *out++ = ')';
+    return out;
+  }
+};
+
+template <typename T, typename Char> struct is_range {
+  static FMT_CONSTEXPR_DECL const bool value =
+      detail::is_range_<T>::value && !detail::is_std_string_like<T>::value &&
+      !detail::is_map<T>::value &&
+      !std::is_convertible<T, std::basic_string<Char>>::value &&
+      !std::is_constructible<detail::std_string_view<Char>, T>::value;
+};
+
+template <typename T, typename Char>
+struct formatter<
+    T, Char,
+    enable_if_t<
+        fmt::is_range<T, Char>::value
+// Workaround a bug in MSVC 2019 and earlier.
+#if !FMT_MSC_VER
+        && (is_formattable<detail::value_type<T>, Char>::value ||
+            detail::has_fallback_formatter<detail::value_type<T>, Char>::value)
+#endif
+        >> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <
+      typename FormatContext, typename U,
+      FMT_ENABLE_IF(
+          std::is_same<U, conditional_t<detail::has_const_begin_end<T>::value,
+                                        const T, T>>::value)>
+  auto format(U& range, FormatContext& ctx) -> decltype(ctx.out()) {
+#ifdef FMT_DEPRECATED_BRACED_RANGES
+    Char prefix = '{';
+    Char postfix = '}';
+#else
+    Char prefix = detail::is_set<T>::value ? '{' : '[';
+    Char postfix = detail::is_set<T>::value ? '}' : ']';
+#endif
+    auto out = ctx.out();
+    *out++ = prefix;
+    int i = 0;
+    auto it = std::begin(range);
+    auto end = std::end(range);
+    for (; it != end; ++it) {
+      if (i > 0) out = detail::write_delimiter(out);
+      out = detail::write_range_entry<Char>(out, *it);
+      ++i;
+    }
+    *out++ = postfix;
+    return out;
+  }
+};
+
+template <typename T, typename Char>
+struct formatter<
+    T, Char,
+    enable_if_t<
+        detail::is_map<T>::value
+// Workaround a bug in MSVC 2019 and earlier.
+#if !FMT_MSC_VER
+        && (is_formattable<detail::value_type<T>, Char>::value ||
+            detail::has_fallback_formatter<detail::value_type<T>, Char>::value)
+#endif
+        >> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <
+      typename FormatContext, typename U,
+      FMT_ENABLE_IF(
+          std::is_same<U, conditional_t<detail::has_const_begin_end<T>::value,
+                                        const T, T>>::value)>
+  auto format(U& map, FormatContext& ctx) -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    *out++ = '{';
+    int i = 0;
+    for (const auto& item : map) {
+      if (i > 0) out = detail::write_delimiter(out);
+      out = detail::write_range_entry<Char>(out, item.first);
+      *out++ = ':';
+      *out++ = ' ';
+      out = detail::write_range_entry<Char>(out, item.second);
+      ++i;
+    }
+    *out++ = '}';
+    return out;
+  }
+};
+
+template <typename Char, typename... T> struct tuple_join_view : detail::view {
+  const std::tuple<T...>& tuple;
+  basic_string_view<Char> sep;
+
+  tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
+      : tuple(t), sep{s} {}
+};
+
+template <typename Char, typename... T>
+using tuple_arg_join = tuple_join_view<Char, T...>;
+
+// Define FMT_TUPLE_JOIN_SPECIFIERS to enable experimental format specifiers
+// support in tuple_join. It is disabled by default because of issues with
+// the dynamic width and precision.
+#ifndef FMT_TUPLE_JOIN_SPECIFIERS
+#  define FMT_TUPLE_JOIN_SPECIFIERS 0
+#endif
+
+template <typename Char, typename... T>
+struct formatter<tuple_join_view<Char, T...>, Char> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return do_parse(ctx, std::integral_constant<size_t, sizeof...(T)>());
+  }
+
+  template <typename FormatContext>
+  auto format(const tuple_join_view<Char, T...>& value,
+              FormatContext& ctx) const -> typename FormatContext::iterator {
+    return do_format(value, ctx,
+                     std::integral_constant<size_t, sizeof...(T)>());
+  }
+
+ private:
+  std::tuple<formatter<typename std::decay<T>::type, Char>...> formatters_;
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+                              std::integral_constant<size_t, 0>)
+      -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename ParseContext, size_t N>
+  FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+                              std::integral_constant<size_t, N>)
+      -> decltype(ctx.begin()) {
+    auto end = ctx.begin();
+#if FMT_TUPLE_JOIN_SPECIFIERS
+    end = std::get<sizeof...(T) - N>(formatters_).parse(ctx);
+    if (N > 1) {
+      auto end1 = do_parse(ctx, std::integral_constant<size_t, N - 1>());
+      if (end != end1)
+        FMT_THROW(format_error("incompatible format specs for tuple elements"));
+    }
+#endif
+    return end;
+  }
+
+  template <typename FormatContext>
+  auto do_format(const tuple_join_view<Char, T...>&, FormatContext& ctx,
+                 std::integral_constant<size_t, 0>) const ->
+      typename FormatContext::iterator {
+    return ctx.out();
+  }
+
+  template <typename FormatContext, size_t N>
+  auto do_format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
+                 std::integral_constant<size_t, N>) const ->
+      typename FormatContext::iterator {
+    auto out = std::get<sizeof...(T) - N>(formatters_)
+                   .format(std::get<sizeof...(T) - N>(value.tuple), ctx);
+    if (N > 1) {
+      out = std::copy(value.sep.begin(), value.sep.end(), out);
+      ctx.advance_to(out);
+      return do_format(value, ctx, std::integral_constant<size_t, N - 1>());
+    }
+    return out;
+  }
+};
+
+FMT_MODULE_EXPORT_BEGIN
+
+/**
+  \rst
+  Returns an object that formats `tuple` with elements separated by `sep`.
+
+  **Example**::
+
+    std::tuple<int, char> t = {1, 'a'};
+    fmt::print("{}", fmt::join(t, ", "));
+    // Output: "1, a"
+  \endrst
+ */
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
+    -> tuple_join_view<char, T...> {
+  return {tuple, sep};
+}
+
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple,
+                        basic_string_view<wchar_t> sep)
+    -> tuple_join_view<wchar_t, T...> {
+  return {tuple, sep};
+}
+
+/**
+  \rst
+  Returns an object that formats `initializer_list` with elements separated by
+  `sep`.
+
+  **Example**::
+
+    fmt::print("{}", fmt::join({1, 2, 3}, ", "));
+    // Output: "1, 2, 3"
+  \endrst
+ */
+template <typename T>
+auto join(std::initializer_list<T> list, string_view sep)
+    -> join_view<const T*, const T*> {
+  return join(std::begin(list), std::end(list), sep);
+}
+
+FMT_MODULE_EXPORT_END
+FMT_END_NAMESPACE
+
+#endif  // FMT_RANGES_H_
diff --git a/contrib/fmt-8.0.1/include/fmt/xchar.h b/contrib/fmt-8.1.1/include/fmt/xchar.h
similarity index 97%
rename from contrib/fmt-8.0.1/include/fmt/xchar.h
rename to contrib/fmt-8.1.1/include/fmt/xchar.h
index a0dd032f16..55825077f8 100644
--- a/contrib/fmt-8.0.1/include/fmt/xchar.h
+++ b/contrib/fmt-8.1.1/include/fmt/xchar.h
@@ -5,8 +5,8 @@
 //
 // For the license information refer to format.h.
 
-#ifndef FMT_WCHAR_H_
-#define FMT_WCHAR_H_
+#ifndef FMT_XCHAR_H_
+#define FMT_XCHAR_H_
 
 #include <cwchar>
 #include <tuple>
@@ -217,11 +217,11 @@ inline void vprint(wstring_view fmt, wformat_args args) {
 
 template <typename... T>
 void print(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
-  return vprint(f, wstring_view(fmt), make_wformat_args(args...));
+  return vprint(f, wstring_view(fmt), fmt::make_wformat_args(args...));
 }
 
 template <typename... T> void print(wformat_string<T...> fmt, T&&... args) {
-  return vprint(wstring_view(fmt), make_wformat_args(args...));
+  return vprint(wstring_view(fmt), fmt::make_wformat_args(args...));
 }
 
 /**
@@ -233,4 +233,4 @@ template <typename T> inline auto to_wstring(const T& value) -> std::wstring {
 FMT_MODULE_EXPORT_END
 FMT_END_NAMESPACE
 
-#endif  // FMT_WCHAR_H_
+#endif  // FMT_XCHAR_H_
diff --git a/contrib/fmt-8.0.1/src/fmt.cc b/contrib/fmt-8.1.1/src/fmt.cc
similarity index 99%
rename from contrib/fmt-8.0.1/src/fmt.cc
rename to contrib/fmt-8.1.1/src/fmt.cc
index d0d6e7fb34..80e77e26af 100644
--- a/contrib/fmt-8.0.1/src/fmt.cc
+++ b/contrib/fmt-8.1.1/src/fmt.cc
@@ -79,7 +79,6 @@ export module fmt;
 #define FMT_END_DETAIL_NAMESPACE \
   }                              \
   export {
-
 // all library-provided declarations and definitions
 // must be in the module purview to be exported
 #include "fmt/args.h"
diff --git a/contrib/fmt-8.0.1/src/format.cc b/contrib/fmt-8.1.1/src/format.cc
similarity index 52%
rename from contrib/fmt-8.0.1/src/format.cc
rename to contrib/fmt-8.1.1/src/format.cc
index 66925b4210..ecb8cc79a6 100644
--- a/contrib/fmt-8.0.1/src/format.cc
+++ b/contrib/fmt-8.1.1/src/format.cc
@@ -10,6 +10,52 @@
 FMT_BEGIN_NAMESPACE
 namespace detail {
 
+// DEPRECATED!
+template <typename T = void> struct basic_data {
+  FMT_API static constexpr const char digits[100][2] = {
+      {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'},
+      {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'},
+      {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'},
+      {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'},
+      {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'},
+      {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'},
+      {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'},
+      {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'},
+      {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'},
+      {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'},
+      {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'},
+      {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'},
+      {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'},
+      {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'},
+      {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'},
+      {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'},
+      {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}};
+  FMT_API static constexpr const char hex_digits[] = "0123456789abcdef";
+  FMT_API static constexpr const char signs[4] = {0, '-', '+', ' '};
+  FMT_API static constexpr const char left_padding_shifts[5] = {31, 31, 0, 1,
+                                                                0};
+  FMT_API static constexpr const char right_padding_shifts[5] = {0, 31, 0, 1,
+                                                                 0};
+  FMT_API static constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
+                                                         0x1000000u | ' '};
+};
+
+#ifdef FMT_SHARED
+// Required for -flto, -fivisibility=hidden and -shared to work
+extern template struct basic_data<void>;
+#endif
+
+#if __cplusplus < 201703L
+// DEPRECATED! These are here only for ABI compatiblity.
+template <typename T> constexpr const char basic_data<T>::digits[][2];
+template <typename T> constexpr const char basic_data<T>::hex_digits[];
+template <typename T> constexpr const char basic_data<T>::signs[];
+template <typename T> constexpr const char basic_data<T>::left_padding_shifts[];
+template <typename T>
+constexpr const char basic_data<T>::right_padding_shifts[];
+template <typename T> constexpr const unsigned basic_data<T>::prefixes[];
+#endif
+
 template <typename T>
 int format_float(char* buf, std::size_t size, const char* format, int precision,
                  T value) {
diff --git a/contrib/fmt-8.0.1/src/os.cc b/contrib/fmt-8.1.1/src/os.cc
similarity index 98%
rename from contrib/fmt-8.0.1/src/os.cc
rename to contrib/fmt-8.1.1/src/os.cc
index 934629d71b..04b4dc5060 100644
--- a/contrib/fmt-8.0.1/src/os.cc
+++ b/contrib/fmt-8.1.1/src/os.cc
@@ -26,19 +26,17 @@
 #    endif
 #    include <io.h>
 
-#    define O_CREAT _O_CREAT
-#    define O_TRUNC _O_TRUNC
-
 #    ifndef S_IRUSR
 #      define S_IRUSR _S_IREAD
 #    endif
-
 #    ifndef S_IWUSR
 #      define S_IWUSR _S_IWRITE
 #    endif
-
-#    ifdef __MINGW32__
-#      define _SH_DENYNO 0x40
+#    ifndef S_IRGRP
+#      define S_IRGRP 0
+#    endif
+#    ifndef S_IROTH
+#      define S_IROTH 0
 #    endif
 #  endif  // _WIN32
 #endif    // FMT_USE_FCNTL
@@ -213,7 +211,10 @@ int buffered_file::fileno() const {
 
 #if FMT_USE_FCNTL
 file::file(cstring_view path, int oflag) {
-  int mode = S_IRUSR | S_IWUSR;
+#  ifdef _WIN32
+  using mode_t = int;
+#  endif
+  mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
 #  if defined(_WIN32) && !defined(__MINGW32__)
   fd_ = -1;
   FMT_POSIX_CALL(sopen_s(&fd_, path.c_str(), oflag, _SH_DENYNO, mode));
diff --git a/contrib/fmt-8.0.1/support/cmake/FindSetEnv.cmake b/contrib/fmt-8.1.1/support/cmake/FindSetEnv.cmake
similarity index 100%
rename from contrib/fmt-8.0.1/support/cmake/FindSetEnv.cmake
rename to contrib/fmt-8.1.1/support/cmake/FindSetEnv.cmake
diff --git a/contrib/fmt-8.0.1/support/cmake/JoinPaths.cmake b/contrib/fmt-8.1.1/support/cmake/JoinPaths.cmake
similarity index 100%
rename from contrib/fmt-8.0.1/support/cmake/JoinPaths.cmake
rename to contrib/fmt-8.1.1/support/cmake/JoinPaths.cmake
diff --git a/contrib/fmt-8.0.1/support/cmake/cxx14.cmake b/contrib/fmt-8.1.1/support/cmake/cxx14.cmake
similarity index 100%
rename from contrib/fmt-8.0.1/support/cmake/cxx14.cmake
rename to contrib/fmt-8.1.1/support/cmake/cxx14.cmake
diff --git a/contrib/fmt-8.0.1/support/cmake/fmt-config.cmake.in b/contrib/fmt-8.1.1/support/cmake/fmt-config.cmake.in
similarity index 100%
rename from contrib/fmt-8.0.1/support/cmake/fmt-config.cmake.in
rename to contrib/fmt-8.1.1/support/cmake/fmt-config.cmake.in
diff --git a/contrib/fmt-8.0.1/support/cmake/fmt.pc.in b/contrib/fmt-8.1.1/support/cmake/fmt.pc.in
similarity index 100%
rename from contrib/fmt-8.0.1/support/cmake/fmt.pc.in
rename to contrib/fmt-8.1.1/support/cmake/fmt.pc.in

From 1a5bf25b5c8f8fc1833f90bc57f3af7e35e3800e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 4 Mar 2022 10:42:45 -0800
Subject: [PATCH 155/210] Update code base to use fmt 8.1.1

---
 bindings/flow/fdb_flow.actor.cpp                           | 2 +-
 documentation/tutorial/tutorial.actor.cpp                  | 2 +-
 fdbbackup/backup.actor.cpp                                 | 2 +-
 fdbcli/AdvanceVersionCommand.actor.cpp                     | 2 +-
 fdbcli/ChangeFeedCommand.actor.cpp                         | 2 +-
 fdbcli/MaintenanceCommand.actor.cpp                        | 2 +-
 fdbcli/SetClassCommand.actor.cpp                           | 2 +-
 fdbcli/fdbcli.actor.cpp                                    | 2 +-
 fdbclient/BackupContainer.actor.cpp                        | 2 +-
 fdbclient/BackupContainerFileSystem.h                      | 2 +-
 fdbclient/BlobGranuleFiles.cpp                             | 2 +-
 fdbclient/BlobGranuleReader.actor.cpp                      | 2 +-
 fdbclient/FileBackupAgent.actor.cpp                        | 2 +-
 fdbclient/ManagementAPI.actor.cpp                          | 2 +-
 fdbclient/NativeAPI.actor.cpp                              | 2 +-
 fdbrpc/sim2.actor.cpp                                      | 2 +-
 fdbserver/BlobManager.actor.cpp                            | 2 +-
 fdbserver/BlobWorker.actor.cpp                             | 2 +-
 fdbserver/KeyValueStoreSQLite.actor.cpp                    | 2 +-
 fdbserver/Status.actor.cpp                                 | 2 +-
 fdbserver/VersionedBTree.actor.cpp                         | 2 +-
 fdbserver/networktest.actor.cpp                            | 2 +-
 fdbserver/workloads/ApiWorkload.actor.cpp                  | 2 +-
 fdbserver/workloads/AsyncFileCorrectness.actor.cpp         | 2 +-
 fdbserver/workloads/BlobGranuleVerifier.actor.cpp          | 2 +-
 fdbserver/workloads/DiskDurabilityTest.actor.cpp           | 2 +-
 fdbserver/workloads/KVStoreTest.actor.cpp                  | 2 +-
 fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp | 2 +-
 fdbserver/workloads/SlowTaskWorkload.actor.cpp             | 2 +-
 flow/DeterministicRandom.cpp                               | 2 +-
 flow/IndexedSet.cpp                                        | 2 +-
 flow/Platform.actor.cpp                                    | 2 +-
 32 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp
index b7013c24ce..bba7da080f 100644
--- a/bindings/flow/fdb_flow.actor.cpp
+++ b/bindings/flow/fdb_flow.actor.cpp
@@ -24,7 +24,7 @@
 #include <stdio.h>
 #include <cinttypes>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/SystemMonitor.h"
 #include "flow/TLSConfig.actor.h"
diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index 8fdb32b0c2..a42677f0ee 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -19,7 +19,7 @@
 * limitations under the License.
 */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/flow.h"
 #include "flow/Platform.h"
 #include "flow/DeterministicRandom.h"
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index e3cdc01ce7..843a86a7db 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbbackup/BackupTLSConfig.h"
 #include "fdbclient/JsonBuilder.h"
 #include "flow/Arena.h"
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp
index 3e6e4de22d..248c742c88 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbcli/fdbcli.actor.h"
 
 #include "fdbclient/IClientApi.h"
diff --git a/fdbcli/ChangeFeedCommand.actor.cpp b/fdbcli/ChangeFeedCommand.actor.cpp
index 94415be921..0cebc508ec 100644
--- a/fdbcli/ChangeFeedCommand.actor.cpp
+++ b/fdbcli/ChangeFeedCommand.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 
 #include "fdbcli/fdbcli.actor.h"
 
diff --git a/fdbcli/MaintenanceCommand.actor.cpp b/fdbcli/MaintenanceCommand.actor.cpp
index f61cbb5d5c..abc8fc355e 100644
--- a/fdbcli/MaintenanceCommand.actor.cpp
+++ b/fdbcli/MaintenanceCommand.actor.cpp
@@ -21,7 +21,7 @@
 #include <cinttypes>
 
 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 
 #include "fdbcli/fdbcli.actor.h"
 
diff --git a/fdbcli/SetClassCommand.actor.cpp b/fdbcli/SetClassCommand.actor.cpp
index 7d91104743..a7b3cf2789 100644
--- a/fdbcli/SetClassCommand.actor.cpp
+++ b/fdbcli/SetClassCommand.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 
 #include "fdbcli/fdbcli.actor.h"
 
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index f7720f9efb..069692346f 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/FDBTypes.h"
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index bfef037be5..ab511a7393 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -22,7 +22,7 @@
 #include <ostream>
 
 // FIXME: Trim this down
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/Platform.actor.h"
 #include "fdbclient/AsyncTaskThread.h"
 #include "fdbclient/BackupContainer.h"
diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h
index 8eae667c0f..8f458df010 100644
--- a/fdbclient/BackupContainerFileSystem.h
+++ b/fdbclient/BackupContainerFileSystem.h
@@ -22,7 +22,7 @@
 #define FDBCLIENT_BACKUP_CONTAINER_FILESYSTEM_H
 #pragma once
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/FDBTypes.h"
 #include "flow/Trace.h"
diff --git a/fdbclient/BlobGranuleFiles.cpp b/fdbclient/BlobGranuleFiles.cpp
index 6779e2da25..f30e8190bf 100644
--- a/fdbclient/BlobGranuleFiles.cpp
+++ b/fdbclient/BlobGranuleFiles.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/serialize.h"
 #include "fdbclient/BlobGranuleFiles.h"
 #include "fdbclient/SystemData.h" // for allKeys unit test - could remove
diff --git a/fdbclient/BlobGranuleReader.actor.cpp b/fdbclient/BlobGranuleReader.actor.cpp
index 6e96507faf..e07f100e32 100644
--- a/fdbclient/BlobGranuleReader.actor.cpp
+++ b/fdbclient/BlobGranuleReader.actor.cpp
@@ -21,7 +21,7 @@
 #include <map>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/AsyncFileS3BlobStore.actor.h"
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/BlobGranuleFiles.h"
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 251e0ab9a4..17e7e6c2cb 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/DatabaseContext.h"
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index bd016538a7..ad98d7fdfb 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -22,7 +22,7 @@
 #include <string>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/Knobs.h"
 #include "flow/Arena.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a21fe9331f..9edd325680 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -28,7 +28,7 @@
 #include <utility>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/FailureMonitor.h"
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index d68652421f..cff1b1d5ad 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -21,7 +21,7 @@
 #include <cinttypes>
 #include <memory>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbrpc/simulator.h"
 #define BOOST_SYSTEM_NO_LIB
 #define BOOST_DATE_TIME_NO_LIB
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index f04360b473..24601dab8b 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -21,7 +21,7 @@
 #include <vector>
 #include <unordered_map>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/BlobWorkerInterface.h"
 #include "fdbclient/KeyRangeMap.h"
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 66b7714854..a8a066f6ea 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -22,7 +22,7 @@
 #include <utility>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/BackupContainerFileSystem.h"
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 4ab37463c5..ea47ef0aa6 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #define SQLITE_THREADSAFE 0 // also in sqlite3.amalgamation.c!
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/crc32c.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/CoroFlow.h"
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index d00b48cf67..fd43618170 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include <cinttypes>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BlobWorkerInterface.h"
 #include "fdbclient/KeyBackedTypes.h"
 #include "fdbserver/Status.h"
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index 6666507512..7c4c21c57e 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbserver/Knobs.h"
 #include "flow/IRandom.h"
diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp
index 3f46250498..e9df6b709d 100644
--- a/fdbserver/networktest.actor.cpp
+++ b/fdbserver/networktest.actor.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/NetworkTest.h"
 #include "flow/Knobs.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
diff --git a/fdbserver/workloads/ApiWorkload.actor.cpp b/fdbserver/workloads/ApiWorkload.actor.cpp
index 822e3122ca..f04c0d59cf 100644
--- a/fdbserver/workloads/ApiWorkload.actor.cpp
+++ b/fdbserver/workloads/ApiWorkload.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include <cinttypes>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/workloads/ApiWorkload.h"
 #include "fdbclient/MultiVersionTransaction.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
diff --git a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp b/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
index 0c92b5098d..9132894b3d 100644
--- a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
+++ b/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
@@ -20,7 +20,7 @@
 
 #include <cinttypes>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/IRandom.h"
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index 2bc2709340..3f27864831 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -22,7 +22,7 @@
 #include <utility>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BlobGranuleReader.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
diff --git a/fdbserver/workloads/DiskDurabilityTest.actor.cpp b/fdbserver/workloads/DiskDurabilityTest.actor.cpp
index 06927e7a6b..d74c17e9ae 100644
--- a/fdbserver/workloads/DiskDurabilityTest.actor.cpp
+++ b/fdbserver/workloads/DiskDurabilityTest.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include <cinttypes>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbrpc/IAsyncFile.h"
 #include "fdbclient/FDBTypes.h"
diff --git a/fdbserver/workloads/KVStoreTest.actor.cpp b/fdbserver/workloads/KVStoreTest.actor.cpp
index 0d10417816..aa3ca8a420 100644
--- a/fdbserver/workloads/KVStoreTest.actor.cpp
+++ b/fdbserver/workloads/KVStoreTest.actor.cpp
@@ -20,7 +20,7 @@
 
 #include <ctime>
 #include <cinttypes>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "flow/ActorCollection.h"
diff --git a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp b/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp
index bfd02399e9..69a9e9667a 100644
--- a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp
+++ b/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp
@@ -20,7 +20,7 @@
 
 #include <cstdint>
 #include <limits>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
diff --git a/fdbserver/workloads/SlowTaskWorkload.actor.cpp b/fdbserver/workloads/SlowTaskWorkload.actor.cpp
index 46342e5d2e..ce48f1770f 100644
--- a/fdbserver/workloads/SlowTaskWorkload.actor.cpp
+++ b/fdbserver/workloads/SlowTaskWorkload.actor.cpp
@@ -20,7 +20,7 @@
 
 #include <cinttypes>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/SignalSafeUnwind.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
diff --git a/flow/DeterministicRandom.cpp b/flow/DeterministicRandom.cpp
index 6b48a83b1b..6357feea40 100644
--- a/flow/DeterministicRandom.cpp
+++ b/flow/DeterministicRandom.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/DeterministicRandom.h"
 
 #include <cstring>
diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp
index a58ff8acae..85e2b4098e 100644
--- a/flow/IndexedSet.cpp
+++ b/flow/IndexedSet.cpp
@@ -21,7 +21,7 @@
 // At the moment, this file just contains tests.  IndexedSet<> is a template
 // and so all the important implementation is in the header file
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/IndexedSet.h"
 #include "flow/IRandom.h"
 #include "flow/ThreadPrimitives.h"
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 3a7da8da52..ffc1cf36ba 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -26,7 +26,7 @@
 #endif
 
 #include <errno.h>
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "flow/Platform.h"
 #include "flow/Platform.actor.h"
 #include "flow/Arena.h"

From 89648769d0ffc0037f44a98e724c89b9e59412e9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 4 Mar 2022 14:03:56 -0800
Subject: [PATCH 156/210] Fix clang format and cmake files for fmt 8.1.1

---
 contrib/fmt-8.1.1/.clang-format  | 9 +--------
 contrib/fmt-8.1.1/CMakeLists.txt | 2 +-
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/contrib/fmt-8.1.1/.clang-format b/contrib/fmt-8.1.1/.clang-format
index df55d340f0..e3845288a2 100644
--- a/contrib/fmt-8.1.1/.clang-format
+++ b/contrib/fmt-8.1.1/.clang-format
@@ -1,8 +1 @@
-# Run manually to reformat a file:
-# clang-format -i --style=file <file>
-Language: Cpp
-BasedOnStyle: Google
-IndentPPDirectives: AfterHash
-IndentCaseLabels: false
-AlwaysBreakTemplateDeclarations: false
-DerivePointerAlignment: false
+DisableFormat: true
diff --git a/contrib/fmt-8.1.1/CMakeLists.txt b/contrib/fmt-8.1.1/CMakeLists.txt
index 3bf80f8022..c392dbd2fc 100644
--- a/contrib/fmt-8.1.1/CMakeLists.txt
+++ b/contrib/fmt-8.1.1/CMakeLists.txt
@@ -241,7 +241,7 @@ else()
   set(FMT_SOURCES src/format.cc)
 endif ()
 
-add_library(fmt ${FMT_SOURCES} ${FMT_HEADERS} README.rst ChangeLog.rst)
+add_library(fmt ${FMT_SOURCES} ${FMT_HEADERS})
 add_library(fmt::fmt ALIAS fmt)
 
 if (HAVE_STRTOD_L)

From 36435af4f5d1f18b6134d081fba7d18a8ccb5dd9 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Sun, 6 Mar 2022 21:52:39 -0800
Subject: [PATCH 157/210] Add support for a limits hint in special keys. Fix a
 typo and resolve some comment formatting weirdness.

---
 fdbclient/ManagementAPI.actor.cpp             |   8 +-
 fdbclient/NativeAPI.actor.cpp                 |  30 ++--
 fdbclient/SpecialKeySpace.actor.cpp           | 170 ++++++++++++------
 fdbclient/SpecialKeySpace.actor.h             | 141 +++++++++++----
 .../SpecialKeySpaceCorrectness.actor.cpp      |   8 +-
 5 files changed, 246 insertions(+), 111 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index ad98d7fdfb..32ebe70317 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1368,9 +1368,9 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 				for (auto& s : servers) {
 					if (!s.isValid()) {
 						if (failed) {
-							ryw.clear(SpecialKeySpace::getManamentApiCommandRange("failed"));
+							ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failed"));
 						} else {
-							ryw.clear(SpecialKeySpace::getManamentApiCommandRange("exclude"));
+							ryw.clear(SpecialKeySpace::getManagementApiCommandRange("exclude"));
 						}
 					} else {
 						Key addr =
@@ -1470,9 +1470,9 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				if (includeAll) {
 					if (failed) {
-						ryw.clear(SpecialKeySpace::getManamentApiCommandRange("failedlocality"));
+						ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failedlocality"));
 					} else {
-						ryw.clear(SpecialKeySpace::getManamentApiCommandRange("excludedlocality"));
+						ryw.clear(SpecialKeySpace::getManagementApiCommandRange("excludedlocality"));
 					}
 				} else {
 					for (const auto& l : localities) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9edd325680..9169dd62f9 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -1123,7 +1123,9 @@ ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord
 ACTOR Future<Optional<Value>> getJSON(Database db);
 
 struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl {
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override {
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override {
 		if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
 			Key prefix = Key(getKeyRange().begin);
 			return map(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()),
@@ -1147,7 +1149,9 @@ struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl {
 };
 
 struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl {
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override {
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override {
 		ASSERT(kr.contains(k));
 		return map(f(ryw), [k = k](Optional<Value> v) {
 			RangeResult result;
@@ -1169,7 +1173,9 @@ private:
 class HealthMetricsRangeImpl : public SpecialKeyRangeAsyncImpl {
 public:
 	explicit HealthMetricsRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRangeRef kr) {
@@ -1254,7 +1260,9 @@ ACTOR static Future<RangeResult> healthMetricsGetRangeActor(ReadYourWritesTransa
 
 HealthMetricsRangeImpl::HealthMetricsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
 
-Future<RangeResult> HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                     KeyRangeRef kr,
+                                                     GetRangeLimits limitsHint) const {
 	return healthMetricsGetRangeActor(ryw, kr);
 }
 
@@ -1353,19 +1361,19 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 		registerSpecialKeySpaceModule(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
-		    std::make_unique<ExcludeServersRangeImpl>(SpecialKeySpace::getManamentApiCommandRange("exclude")));
+		    std::make_unique<ExcludeServersRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("exclude")));
 		registerSpecialKeySpaceModule(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
-		    std::make_unique<FailedServersRangeImpl>(SpecialKeySpace::getManamentApiCommandRange("failed")));
+		    std::make_unique<FailedServersRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("failed")));
 		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT,
 		                              SpecialKeySpace::IMPLTYPE::READWRITE,
 		                              std::make_unique<ExcludedLocalitiesRangeImpl>(
-		                                  SpecialKeySpace::getManamentApiCommandRange("excludedlocality")));
-		registerSpecialKeySpaceModule(
-		    SpecialKeySpace::MODULE::MANAGEMENT,
-		    SpecialKeySpace::IMPLTYPE::READWRITE,
-		    std::make_unique<FailedLocalitiesRangeImpl>(SpecialKeySpace::getManamentApiCommandRange("failedlocality")));
+		                                  SpecialKeySpace::getManagementApiCommandRange("excludedlocality")));
+		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT,
+		                              SpecialKeySpace::IMPLTYPE::READWRITE,
+		                              std::make_unique<FailedLocalitiesRangeImpl>(
+		                                  SpecialKeySpace::getManagementApiCommandRange("failedlocality")));
 		registerSpecialKeySpaceModule(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index fd1087c576..78f9c93e50 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -146,8 +146,11 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
                                                  ReadYourWritesTransaction* ryw,
                                                  KeySelector* ks,
                                                  Optional<RangeResult>* cache) {
-	ASSERT(!ks->orEqual); // should be removed before calling
-	ASSERT(ks->offset != 1); // never being called if KeySelector is already normalized
+	// should be removed before calling
+	ASSERT(!ks->orEqual);
+
+	// never being called if KeySelector is already normalized
+	ASSERT(ks->offset != 1);
 
 	state Key startKey(skrImpl->getKeyRange().begin);
 	state Key endKey(skrImpl->getKeyRange().end);
@@ -162,7 +165,9 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 		if (skrImpl->getKeyRange().contains(ks->getKey()))
 			startKey = ks->getKey();
 	}
-	ASSERT(startKey < endKey); // Note : startKey never equals endKey here
+
+	// Note : startKey never equals endKey here
+	ASSERT(startKey < endKey);
 
 	TraceEvent(SevDebug, "NormalizeKeySelector")
 	    .detail("OriginalKey", ks->getKey())
@@ -170,12 +175,14 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 	    .detail("SpecialKeyRangeStart", skrImpl->getKeyRange().begin)
 	    .detail("SpecialKeyRangeEnd", skrImpl->getKeyRange().end);
 
+	GetRangeLimits limitsHint(ks->offset >= 1 ? ks->offset : 1 - ks->offset);
+
 	if (skrImpl->isAsync()) {
 		const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(skrImpl);
-		RangeResult result_ = wait(ptr->getRange(ryw, KeyRangeRef(startKey, endKey), cache));
+		RangeResult result_ = wait(ptr->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint, cache));
 		result = result_;
 	} else {
-		RangeResult result_ = wait(skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey)));
+		RangeResult result_ = wait(skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint));
 		result = result_;
 	}
 
@@ -197,9 +204,8 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 			ks->setKey(KeyRef(ks->arena(), result[ks->offset - 1].key));
 			ks->offset = 1;
 		} else {
-			ks->setKey(KeyRef(
-			    ks->arena(),
-			    keyAfter(result[result.size() - 1].key))); // TODO : the keyAfter will just return if key == \xff\xff
+			// TODO : the keyAfter will just return if key == \xff\xff
+			ks->setKey(KeyRef(ks->arena(), keyAfter(result[result.size() - 1].key)));
 			ks->offset -= result.size();
 		}
 	}
@@ -277,8 +283,10 @@ SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool
 	// Default begin of KeyRangeMap is Key(), insert the range to update start key
 	readImpls.insert(range, nullptr);
 	writeImpls.insert(range, nullptr);
-	if (!testOnly)
-		modulesBoundaryInit(); // testOnly is used in the correctness workload
+	if (!testOnly) {
+		// testOnly is used in the correctness workload
+		modulesBoundaryInit();
+	}
 }
 
 void SpecialKeySpace::modulesBoundaryInit() {
@@ -376,10 +384,10 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
 			if (iter->value()->isAsync() && cache.present()) {
 				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
-				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), &cache));
+				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
 				pairs = pairs_;
 			} else {
-				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd)));
+				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits));
 				pairs = pairs_;
 			}
 			result.arena().dependsOn(pairs.arena());
@@ -407,10 +415,10 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
 			if (iter->value()->isAsync() && cache.present()) {
 				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
-				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), &cache));
+				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
 				pairs = pairs_;
 			} else {
-				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd)));
+				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits));
 				pairs = pairs_;
 			}
 			result.arena().dependsOn(pairs.arena());
@@ -543,15 +551,17 @@ void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module,
 		ASSERT(normalKeys.contains(kr));
 	} else {
 		ASSERT(moduleToBoundary.at(module).contains(kr));
-		ASSERT(validateSnakeCaseNaming(kr.begin) &&
-		       validateSnakeCaseNaming(kr.end)); // validate keys follow snake case naming style
+		// validate keys follow snake case naming style
+		ASSERT(validateSnakeCaseNaming(kr.begin) && validateSnakeCaseNaming(kr.end));
 	}
 	// make sure the registered range is not overlapping with existing ones
 	// Note: kr.end should not be the same as another range's begin, although it should work even they are the same
 	for (auto iter = readImpls.rangeContaining(kr.begin); true; ++iter) {
 		ASSERT(iter->value() == nullptr);
-		if (iter == readImpls.rangeContaining(kr.end))
-			break; // Note: relax the condition that the end can be another range's start, if needed
+		if (iter == readImpls.rangeContaining(kr.end)) {
+			// Note: relax the condition that the end can be another range's start, if needed
+			break;
+		}
 	}
 	readImpls.insert(kr, impl);
 	// if rw, it means the module can do both read and write
@@ -601,7 +611,7 @@ ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction*
 		Optional<std::string> msg = wait((*it)->commit(ryw));
 		if (msg.present()) {
 			ryw->setSpecialKeySpaceErrorMsg(msg.get());
-			TraceEvent(SevDebug, "SpecialKeySpaceManagemetnAPIError")
+			TraceEvent(SevDebug, "SpecialKeySpaceManagementAPIError")
 			    .detail("Reason", msg.get())
 			    .detail("Range", (*it)->getKeyRange().toString());
 			throw special_keys_api_failure();
@@ -616,7 +626,9 @@ Future<Void> SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) {
 
 SKSCTestImpl::SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw,
+                                           KeyRangeRef kr,
+                                           GetRangeLimits limitsHint) const {
 	ASSERT(range.contains(kr));
 	auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY);
 	// all keys are written to RYW, since GRV is set, the read should happen locally
@@ -639,19 +651,25 @@ ACTOR static Future<RangeResult> getReadConflictRangeImpl(ReadYourWritesTransact
 	return ryw->getReadConflictRangeIntersecting(kr);
 }
 
-Future<RangeResult> ReadConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ReadConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                    KeyRangeRef kr,
+                                                    GetRangeLimits limitsHint) const {
 	return getReadConflictRangeImpl(ryw, kr);
 }
 
 WriteConflictRangeImpl::WriteConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-Future<RangeResult> WriteConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> WriteConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                     KeyRangeRef kr,
+                                                     GetRangeLimits limitsHint) const {
 	return ryw->getWriteConflictRangeIntersecting(kr);
 }
 
 ConflictingKeysImpl::ConflictingKeysImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-Future<RangeResult> ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                  KeyRangeRef kr,
+                                                  GetRangeLimits limitsHint) const {
 	RangeResult result;
 	if (ryw->getTransactionState()->conflictingKeys) {
 		auto krMapPtr = ryw->getTransactionState()->conflictingKeys.get();
@@ -702,7 +720,9 @@ ACTOR Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw,
 
 DDStatsRangeImpl::DDStatsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
 
-Future<RangeResult> DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                               KeyRangeRef kr,
+                                               GetRangeLimits limitsHint) const {
 	return ddMetricsGetRangeActor(ryw, kr);
 }
 
@@ -715,7 +735,9 @@ Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string&
 
 ManagementCommandsOptionsImpl::ManagementCommandsOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> ManagementCommandsOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ManagementCommandsOptionsImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                            KeyRangeRef kr,
+                                                            GetRangeLimits limitsHint) const {
 	RangeResult result;
 	// Since we only have limit number of options, a brute force loop here is enough
 	for (const auto& option : SpecialKeySpace::getManagementApiOptionsSet()) {
@@ -822,7 +844,9 @@ ACTOR Future<RangeResult> rwModuleWithMappingGetRangeActor(ReadYourWritesTransac
 
 ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                      KeyRangeRef kr,
+                                                      GetRangeLimits limitsHint) const {
 	ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	return rwModuleWithMappingGetRangeActor(ryw, this, kr);
 }
@@ -847,8 +871,8 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw,
                               std::vector<AddressExclusion>& addresses,
                               std::set<AddressExclusion>& exclusions,
                               Optional<std::string>& msg) {
-	KeyRangeRef range = failed ? SpecialKeySpace::getManamentApiCommandRange("failed")
-	                           : SpecialKeySpace::getManamentApiCommandRange("exclude");
+	KeyRangeRef range = failed ? SpecialKeySpace::getManagementApiCommandRange("failed")
+	                           : SpecialKeySpace::getManagementApiCommandRange("exclude");
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
 	auto iter = ranges.begin();
 	while (iter != ranges.end()) {
@@ -1009,7 +1033,7 @@ void includeServers(ReadYourWritesTransaction* ryw) {
 	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	// for exluded servers
 	auto ranges =
-	    ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("exclude"));
+	    ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManagementApiCommandRange("exclude"));
 	auto iter = ranges.begin();
 	Transaction& tr = ryw->getTransaction();
 	while (iter != ranges.end()) {
@@ -1022,7 +1046,7 @@ void includeServers(ReadYourWritesTransaction* ryw) {
 		++iter;
 	}
 	// for failed servers
-	ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("failed"));
+	ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManagementApiCommandRange("failed"));
 	iter = ranges.begin();
 	while (iter != ranges.end()) {
 		auto entry = iter->value();
@@ -1063,7 +1087,9 @@ Future<Optional<std::string>> ExcludeServersRangeImpl::commit(ReadYourWritesTran
 
 FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                     KeyRangeRef kr,
+                                                     GetRangeLimits limitsHint) const {
 	ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	return rwModuleWithMappingGetRangeActor(ryw, this, kr);
 }
@@ -1144,7 +1170,9 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 
 ExclusionInProgressRangeImpl::ExclusionInProgressRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
 
-Future<RangeResult> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                           KeyRangeRef kr,
+                                                           GetRangeLimits limitsHint) const {
 	return ExclusionInProgressActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -1211,7 +1239,9 @@ ACTOR Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransa
 
 ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                    KeyRangeRef kr,
+                                                    GetRangeLimits limitsHint) const {
 	return getProcessClassActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -1287,7 +1317,9 @@ ACTOR Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction*
 
 ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-Future<RangeResult> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                          KeyRangeRef kr,
+                                                          GetRangeLimits limitsHint) const {
 	return getProcessClassSourceActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -1305,7 +1337,9 @@ ACTOR Future<RangeResult> getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyR
 
 LockDatabaseImpl::LockDatabaseImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw,
+                                               KeyRangeRef kr,
+                                               GetRangeLimits limitsHint) const {
 	// single key range, the queried range should always be the same as the underlying range
 	ASSERT(kr == getKeyRange());
 	auto lockEntry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")];
@@ -1386,7 +1420,9 @@ ACTOR Future<RangeResult> getConsistencyCheckKeyActor(ReadYourWritesTransaction*
 
 ConsistencyCheckImpl::ConsistencyCheckImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> ConsistencyCheckImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ConsistencyCheckImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                   KeyRangeRef kr,
+                                                   GetRangeLimits limitsHint) const {
 	// single key range, the queried range should always be the same as the underlying range
 	ASSERT(kr == getKeyRange());
 	auto entry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")];
@@ -1419,7 +1455,9 @@ GlobalConfigImpl::GlobalConfigImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {
 // framework within the range specified. The special-key-space getrange
 // function should only be used for informational purposes. All values are
 // returned as strings regardless of their true type.
-Future<RangeResult> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw,
+                                               KeyRangeRef kr,
+                                               GetRangeLimits limitsHint) const {
 	RangeResult result;
 
 	auto& globalConfig = GlobalConfig::globalConfig();
@@ -1540,7 +1578,9 @@ void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key)
 
 TracingOptionsImpl::TracingOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                 KeyRangeRef kr,
+                                                 GetRangeLimits limitsHint) const {
 	RangeResult result;
 	for (const auto& option : SpecialKeySpace::getTracingOptions()) {
 		auto key = getKeyRange().begin.withSuffix(option);
@@ -1599,7 +1639,9 @@ void TracingOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key
 
 CoordinatorsImpl::CoordinatorsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw,
+                                               KeyRangeRef kr,
+                                               GetRangeLimits limitsHint) const {
 	RangeResult result;
 	KeyRef prefix(getKeyRange().begin);
 	auto cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString();
@@ -1781,7 +1823,9 @@ ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransac
 	return res;
 }
 
-Future<RangeResult> CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                   KeyRangeRef kr,
+                                                   GetRangeLimits limitsHint) const {
 	// single key range, the queried range should always be the same as the underlying range
 	ASSERT(kr == getKeyRange());
 	return CoordinatorsAutoImplActor(ryw, kr);
@@ -1802,7 +1846,9 @@ ACTOR static Future<RangeResult> getMinCommitVersionActor(ReadYourWritesTransact
 
 AdvanceVersionImpl::AdvanceVersionImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> AdvanceVersionImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> AdvanceVersionImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                 KeyRangeRef kr,
+                                                 GetRangeLimits limitsHint) const {
 	// single key range, the queried range should always be the same as the underlying range
 	ASSERT(kr == getKeyRange());
 	auto entry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("advanceversion")];
@@ -1871,7 +1917,8 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 	if (kr.contains(sampleRateKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey];
 		if (!ryw->readYourWritesDisabled() && entry.first) {
-			ASSERT(entry.second.present()); // clear is forbidden
+			// clear is forbidden
+			ASSERT(entry.second.present());
 			result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, entry.second.get()));
 		} else {
 			Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSampleRate));
@@ -1890,7 +1937,8 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 	if (kr.contains(txnSizeLimitKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey];
 		if (!ryw->readYourWritesDisabled() && entry.first) {
-			ASSERT(entry.second.present()); // clear is forbidden
+			// clear is forbidden
+			ASSERT(entry.second.present());
 			result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, entry.second.get()));
 		} else {
 			Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSizeLimit));
@@ -1908,7 +1956,9 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 }
 
 // TODO : add limitation on set operation
-Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                  KeyRangeRef kr,
+                                                  GetRangeLimits limitsHint) const {
 	return ClientProfilingGetRangeActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -2195,7 +2245,9 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 	return result;
 }
 
-Future<RangeResult> ActorLineageImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ActorLineageImpl::getRange(ReadYourWritesTransaction* ryw,
+                                               KeyRangeRef kr,
+                                               GetRangeLimits limitsHint) const {
 	return actorLineageGetRangeActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -2208,7 +2260,9 @@ std::string_view to_string_view(StringRef sr) {
 ActorProfilerConf::ActorProfilerConf(KeyRangeRef kr)
   : SpecialKeyRangeRWImpl(kr), config(ProfilerConfig::instance().getConfig()) {}
 
-Future<RangeResult> ActorProfilerConf::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ActorProfilerConf::getRange(ReadYourWritesTransaction* ryw,
+                                                KeyRangeRef kr,
+                                                GetRangeLimits limitsHint) const {
 	RangeResult res;
 	std::string_view begin(to_string_view(kr.begin.removePrefix(range.begin))),
 	    end(to_string_view(kr.end.removePrefix(range.begin)));
@@ -2297,7 +2351,9 @@ ACTOR static Future<RangeResult> MaintenanceGetRangeActor(ReadYourWritesTransact
 	return rywGetRange(ryw, kr, result);
 }
 
-Future<RangeResult> MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw,
+                                              KeyRangeRef kr,
+                                              GetRangeLimits limitsHint) const {
 	return MaintenanceGetRangeActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -2401,7 +2457,9 @@ ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTra
 	return rywGetRange(ryw, kr, result);
 }
 
-Future<RangeResult> DataDistributionImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> DataDistributionImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                   KeyRangeRef kr,
+                                                   GetRangeLimits limitsHint) const {
 	return DataDistributionGetRangeActor(ryw, getKeyRange().begin, kr);
 }
 
@@ -2486,7 +2544,7 @@ void includeLocalities(ReadYourWritesTransaction* ryw) {
 	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	// for excluded localities
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(
-	    SpecialKeySpace::getManamentApiCommandRange("excludedlocality"));
+	    SpecialKeySpace::getManagementApiCommandRange("excludedlocality"));
 	Transaction& tr = ryw->getTransaction();
 	for (auto& iter : ranges) {
 		auto entry = iter.value();
@@ -2498,7 +2556,7 @@ void includeLocalities(ReadYourWritesTransaction* ryw) {
 	}
 	// for failed localities
 	ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(
-	    SpecialKeySpace::getManamentApiCommandRange("failedlocality"));
+	    SpecialKeySpace::getManagementApiCommandRange("failedlocality"));
 	for (auto& iter : ranges) {
 		auto entry = iter.value();
 		if (entry.first && !entry.second.present()) {
@@ -2518,8 +2576,8 @@ bool parseLocalitiesFromKeys(ReadYourWritesTransaction* ryw,
                              std::set<AddressExclusion>& exclusions,
                              std::vector<ProcessData>& workers,
                              Optional<std::string>& msg) {
-	KeyRangeRef range = failed ? SpecialKeySpace::getManamentApiCommandRange("failedlocality")
-	                           : SpecialKeySpace::getManamentApiCommandRange("excludedlocality");
+	KeyRangeRef range = failed ? SpecialKeySpace::getManagementApiCommandRange("failedlocality")
+	                           : SpecialKeySpace::getManagementApiCommandRange("excludedlocality");
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
 	auto iter = ranges.begin();
 	while (iter != ranges.end()) {
@@ -2583,7 +2641,9 @@ ACTOR Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTra
 
 ExcludedLocalitiesRangeImpl::ExcludedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> ExcludedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> ExcludedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                          KeyRangeRef kr,
+                                                          GetRangeLimits limitsHint) const {
 	ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	return rwModuleWithMappingGetRangeActor(ryw, this, kr);
 }
@@ -2610,7 +2670,9 @@ Future<Optional<std::string>> ExcludedLocalitiesRangeImpl::commit(ReadYourWrites
 
 FailedLocalitiesRangeImpl::FailedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-Future<RangeResult> FailedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
+Future<RangeResult> FailedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                        KeyRangeRef kr,
+                                                        GetRangeLimits limitsHint) const {
 	ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	return rwModuleWithMappingGetRangeActor(ryw, this, kr);
 }
diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h
index 1bc985b2fc..13a49b055b 100644
--- a/fdbclient/SpecialKeySpace.actor.h
+++ b/fdbclient/SpecialKeySpace.actor.h
@@ -36,7 +36,9 @@
 class SpecialKeyRangeReadImpl {
 public:
 	// Each derived class only needs to implement this simple version of getRange
-	virtual Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const = 0;
+	virtual Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                                     KeyRangeRef kr,
+	                                     GetRangeLimits limitsHint) const = 0;
 
 	explicit SpecialKeyRangeReadImpl(KeyRangeRef kr) : range(kr) {}
 	KeyRangeRef getKeyRange() const { return range; }
@@ -48,7 +50,8 @@ public:
 	virtual ~SpecialKeyRangeReadImpl() {}
 
 protected:
-	KeyRange range; // underlying key range for this function
+	// underlying key range for this function
+	KeyRange range;
 };
 
 class ManagementAPIError {
@@ -76,8 +79,9 @@ public:
 	virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
 		ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>()));
 	}
-	virtual Future<Optional<std::string>> commit(
-	    ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space
+	// all delayed async operations of writes in special-key-space
+	virtual Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) = 0;
+
 	// Given the special key to write, return the real key that needs to be modified
 	virtual Key decode(const KeyRef& key) const {
 		// Default implementation should never be used
@@ -100,11 +104,16 @@ class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit SpecialKeyRangeAsyncImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override = 0;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override = 0;
 
 	// calling with a cache object to have consistent results if we need to call rpc
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr, Optional<RangeResult>* cache) const {
-		return getRangeAsyncActor(this, ryw, kr, cache);
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint,
+	                             Optional<RangeResult>* cache) const {
+		return getRangeAsyncActor(this, ryw, kr, limitsHint, cache);
 	}
 
 	bool isAsync() const override { return true; }
@@ -112,6 +121,7 @@ public:
 	ACTOR static Future<RangeResult> getRangeAsyncActor(const SpecialKeyRangeReadImpl* skrAyncImpl,
 	                                                    ReadYourWritesTransaction* ryw,
 	                                                    KeyRangeRef kr,
+	                                                    GetRangeLimits limits,
 	                                                    Optional<RangeResult>* cache) {
 		ASSERT(skrAyncImpl->getKeyRange().contains(kr));
 		ASSERT(cache != nullptr);
@@ -119,7 +129,7 @@ public:
 			// For simplicity, every time we need to cache, we read the whole range
 			// Although sometimes the range can be narrowed,
 			// there is not a general way to do it in complicated scenarios
-			RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange()));
+			RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange(), limits));
 			*cache = result_;
 		}
 		const auto& allResults = cache->get();
@@ -193,7 +203,7 @@ public:
 	KeyRangeMap<SpecialKeySpace::MODULE>& getModules() { return modules; }
 	KeyRangeRef getKeyRange() const { return range; }
 	static KeyRangeRef getModuleRange(SpecialKeySpace::MODULE module) { return moduleToBoundary.at(module); }
-	static KeyRangeRef getManamentApiCommandRange(const std::string& command) {
+	static KeyRangeRef getManagementApiCommandRange(const std::string& command) {
 		return managementApiCommandToRange.at(command);
 	}
 	static KeyRef getManagementApiCommandPrefix(const std::string& command) {
@@ -228,13 +238,18 @@ private:
 	KeyRangeMap<SpecialKeyRangeReadImpl*> readImpls;
 	KeyRangeMap<SpecialKeySpace::MODULE> modules;
 	KeyRangeMap<SpecialKeyRangeRWImpl*> writeImpls;
-	KeyRange range; // key space range, (\xff\xff, \xff\xff\xff) in prod and (, \xff) in test
+
+	// key space range, (\xff\xff, \xff\xff\xff) in prod and (, \xff) in test
+	KeyRange range;
 
 	static std::unordered_map<SpecialKeySpace::MODULE, KeyRange> moduleToBoundary;
-	static std::unordered_map<std::string, KeyRange>
-	    managementApiCommandToRange; // management command to its special keys' range
+
+	// management command to its special keys' range
+	static std::unordered_map<std::string, KeyRange> managementApiCommandToRange;
 	static std::unordered_map<std::string, KeyRange> actorLineageApiCommandToRange;
-	static std::set<std::string> options; // "<command>/<option>"
+
+	// "<command>/<option>"
+	static std::set<std::string> options;
 	static std::set<std::string> tracingOptions;
 
 	// Initialize module boundaries, used to handle cross_module_read
@@ -245,7 +260,9 @@ private:
 class SKSCTestImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit SKSCTestImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
@@ -258,31 +275,41 @@ public:
 class ConflictingKeysImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit ConflictingKeysImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class ReadConflictRangeImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit ReadConflictRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class WriteConflictRangeImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit WriteConflictRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class DDStatsRangeImpl : public SpecialKeyRangeAsyncImpl {
 public:
 	explicit DDStatsRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class ManagementCommandsOptionsImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ManagementCommandsOptionsImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
@@ -293,7 +320,9 @@ public:
 class ExcludedLocalitiesRangeImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ExcludedLocalitiesRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Key decode(const KeyRef& key) const override;
 	Key encode(const KeyRef& key) const override;
@@ -304,7 +333,9 @@ public:
 class FailedLocalitiesRangeImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit FailedLocalitiesRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Key decode(const KeyRef& key) const override;
 	Key encode(const KeyRef& key) const override;
@@ -314,7 +345,9 @@ public:
 class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ExcludeServersRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Key decode(const KeyRef& key) const override;
 	Key encode(const KeyRef& key) const override;
@@ -324,7 +357,9 @@ public:
 class FailedServersRangeImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit FailedServersRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Key decode(const KeyRef& key) const override;
 	Key encode(const KeyRef& key) const override;
@@ -334,13 +369,17 @@ public:
 class ExclusionInProgressRangeImpl : public SpecialKeyRangeAsyncImpl {
 public:
 	explicit ExclusionInProgressRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class ProcessClassRangeImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ProcessClassRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
@@ -349,27 +388,35 @@ public:
 class ProcessClassSourceRangeImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit ProcessClassSourceRangeImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class LockDatabaseImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit LockDatabaseImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
 class ConsistencyCheckImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ConsistencyCheckImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
 class GlobalConfigImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit GlobalConfigImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
@@ -379,7 +426,9 @@ public:
 class TracingOptionsImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit TracingOptionsImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
@@ -389,7 +438,9 @@ public:
 class CoordinatorsImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit CoordinatorsImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
@@ -398,20 +449,26 @@ public:
 class CoordinatorsAutoImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit CoordinatorsAutoImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class AdvanceVersionImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit AdvanceVersionImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
 class ClientProfilingImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ClientProfilingImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
@@ -420,7 +477,9 @@ public:
 class ActorLineageImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit ActorLineageImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 };
 
 class ActorProfilerConf : public SpecialKeyRangeRWImpl {
@@ -429,7 +488,9 @@ class ActorProfilerConf : public SpecialKeyRangeRWImpl {
 
 public:
 	explicit ActorProfilerConf(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
 	void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
@@ -439,14 +500,18 @@ public:
 class MaintenanceImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit MaintenanceImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
 class DataDistributionImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit DataDistributionImpl(KeyRangeRef kr);
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index 8f7b2db8dc..98c75c26aa 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -429,8 +429,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// A clear cross two ranges are forbidden
 		try {
 			tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-			tx->clear(KeyRangeRef(SpecialKeySpace::getManamentApiCommandRange("exclude").begin,
-			                      SpecialKeySpace::getManamentApiCommandRange("failed").end));
+			tx->clear(KeyRangeRef(SpecialKeySpace::getManagementApiCommandRange("exclude").begin,
+			                      SpecialKeySpace::getManagementApiCommandRange("failed").end));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -1274,11 +1274,11 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			try {
 				// maintenance
 				RangeResult maintenanceKVs = wait(
-				    tx->getRange(SpecialKeySpace::getManamentApiCommandRange("maintenance"), CLIENT_KNOBS->TOO_MANY));
+				    tx->getRange(SpecialKeySpace::getManagementApiCommandRange("maintenance"), CLIENT_KNOBS->TOO_MANY));
 				// By default, no maintenance is going on
 				ASSERT(!maintenanceKVs.more && !maintenanceKVs.size());
 				// datadistribution
-				RangeResult ddKVs = wait(tx->getRange(SpecialKeySpace::getManamentApiCommandRange("datadistribution"),
+				RangeResult ddKVs = wait(tx->getRange(SpecialKeySpace::getManagementApiCommandRange("datadistribution"),
 				                                      CLIENT_KNOBS->TOO_MANY));
 				// By default, data_distribution/mode := "-1"
 				ASSERT(!ddKVs.more && ddKVs.size() == 1);

From 5fa9d3e1b74d15a13f45fb10936b53f3e739c365 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 18 Feb 2022 21:22:31 -0800
Subject: [PATCH 158/210] Add a tenant parameter to read and commit requests.
 Store a map of all tenants on commit proxy and storage servers. Add an option
 to require tenant mode.

---
 fdbclient/CMakeLists.txt              |   1 +
 fdbclient/CommitProxyInterface.h      |  14 +-
 fdbclient/DatabaseConfiguration.cpp   |   8 +-
 fdbclient/DatabaseConfiguration.h     |   2 +
 fdbclient/FDBTypes.h                  |  37 +++
 fdbclient/ManagementAPI.actor.cpp     |  12 +
 fdbclient/NativeAPI.actor.cpp         |  10 +-
 fdbclient/Schemas.cpp                 |   5 +
 fdbclient/StorageServerInterface.h    | 100 ++++++--
 fdbclient/SystemData.cpp              |  15 ++
 fdbclient/SystemData.h                |   9 +
 fdbclient/Tenant.h                    |  49 ++++
 fdbserver/ApplyMetadataMutation.cpp   | 106 ++++++++-
 fdbserver/ApplyMetadataMutation.h     |   1 +
 fdbserver/ClusterController.actor.cpp |   9 +-
 fdbserver/CommitProxyServer.actor.cpp |  60 ++++-
 fdbserver/ConflictSet.h               |   1 +
 fdbserver/ProxyCommitData.actor.h     |   3 +
 fdbserver/SimulatedCluster.actor.cpp  |   2 +-
 fdbserver/storageserver.actor.cpp     | 329 +++++++++++++++++++++++---
 flow/error_definitions.h              |   4 +
 21 files changed, 700 insertions(+), 77 deletions(-)
 create mode 100644 fdbclient/Tenant.h

diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt
index 8a0350e3a2..1fbb8d532d 100644
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@@ -138,6 +138,7 @@ set(FDBCLIENT_SRCS
   TagThrottle.actor.h
   TaskBucket.actor.cpp
   TaskBucket.h
+  Tenant.h
   TestKnobCollection.cpp
   TestKnobCollection.h
   ThreadSafeTransaction.cpp
diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index 688f6a2504..b0f1dc0e08 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -116,6 +116,8 @@ struct ClientDBInfo {
 	Optional<Value> forward;
 	std::vector<VersionHistory> history;
 
+	TenantMode tenantMode;
+
 	ClientDBInfo() {}
 
 	bool operator==(ClientDBInfo const& r) const { return id == r.id; }
@@ -126,7 +128,7 @@ struct ClientDBInfo {
 		if constexpr (!is_fb_function<Archive>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, grvProxies, commitProxies, id, forward, history);
+		serializer(ar, grvProxies, commitProxies, id, forward, history, tenantMode);
 	}
 };
 
@@ -167,12 +169,16 @@ struct CommitTransactionRequest : TimedRequest {
 	Optional<ClientTrCommitCostEstimation> commitCostEstimation;
 	Optional<TagSet> tagSet;
 
-	CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {}
-	CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {}
+	TenantInfo tenantInfo;
+
+	CommitTransactionRequest() : CommitTransactionRequest(TenantInfo(), SpanID()) {}
+	CommitTransactionRequest(TenantInfo const& tenantInfo, SpanID const& context)
+	  : spanContext(context), flags(0), tenantInfo(tenantInfo) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext);
+		serializer(
+		    ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext, tenantInfo);
 	}
 };
 
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 3328443fc7..d84682953f 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -50,6 +50,7 @@ void DatabaseConfiguration::resetInternal() {
 	perpetualStorageWiggleSpeed = 0;
 	perpetualStorageWiggleLocality = "0";
 	storageMigrationType = StorageMigrationType::DEFAULT;
+	tenantMode = TenantMode::OPTIONAL;
 }
 
 int toInt(ValueRef const& v) {
@@ -209,7 +210,8 @@ bool DatabaseConfiguration::isValid() const {
 	      // We cannot specify regions with three_datacenter replication
 	      (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) &&
 	      isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality) &&
-	      storageMigrationType != StorageMigrationType::UNSET)) {
+	      storageMigrationType != StorageMigrationType::UNSET && tenantMode >= TenantMode::OPTIONAL &&
+	      tenantMode < TenantMode::END)) {
 		return false;
 	}
 	std::set<Key> dcIds;
@@ -402,6 +404,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed;
 	result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality;
 	result["storage_migration_type"] = storageMigrationType.toString();
+	result["tenant_mode"] = tenantMode.toString();
 	return result;
 }
 
@@ -625,6 +628,9 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	} else if (ck == LiteralStringRef("storage_migration_type")) {
 		parse((&type), value);
 		storageMigrationType = (StorageMigrationType::MigrationType)type;
+	} else if (ck == LiteralStringRef("tenant_mode")) {
+		parse((&type), value);
+		tenantMode = (TenantMode::Mode)type;
 	} else if (ck == LiteralStringRef("proxies")) {
 		overwriteProxiesCount();
 	} else {
diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h
index 81bb6f7084..398c4c5b91 100644
--- a/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/DatabaseConfiguration.h
@@ -250,6 +250,8 @@ struct DatabaseConfiguration {
 	// Storage Migration Type
 	StorageMigrationType storageMigrationType;
 
+	TenantMode tenantMode;
+
 	// Excluded servers (no state should be here)
 	bool isExcludedServer(NetworkAddressList) const;
 	bool isExcludedLocality(const LocalityData& locality) const;
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 4c4d1b46e5..c51afa6530 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -30,6 +30,10 @@
 #include "flow/Arena.h"
 #include "flow/flow.h"
 
+#ifdef _WIN32
+#undef OPTIONAL
+#endif
+
 typedef int64_t Version;
 typedef uint64_t LogEpoch;
 typedef uint64_t Sequence;
@@ -1180,6 +1184,39 @@ struct StorageMigrationType {
 	uint32_t type;
 };
 
+struct TenantMode {
+	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones
+	// just before END.
+	enum Mode { OPTIONAL = 0, REQUIRED = 1, END = 2 };
+
+	TenantMode() : mode(OPTIONAL) {}
+	TenantMode(Mode mode) : mode(mode) {
+		if ((uint32_t)mode >= END) {
+			this->mode = OPTIONAL;
+		}
+	}
+	operator Mode() const { return Mode(mode); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, mode);
+	}
+
+	std::string toString() const {
+		switch (mode) {
+		case OPTIONAL:
+			return "optional";
+		case REQUIRED:
+			return "required";
+		default:
+			ASSERT(false);
+		}
+		return "";
+	}
+
+	uint32_t mode;
+};
+
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
 	int pos = locality.find(':');
 	// locality should be either 0 or in the format '<non_empty_string>:<non_empty_string>'
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 32ebe70317..6f8f9350c7 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -174,6 +174,18 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			}
 			out[p + key] = format("%d", type);
 		}
+		if (key == "tenant_mode") {
+			TenantMode tenantMode;
+			if (value == "optional") {
+				tenantMode = TenantMode::OPTIONAL;
+			} else if (value == "required") {
+				tenantMode = TenantMode::REQUIRED;
+			} else {
+				printf("Error: Only optional|required are valid for tenant_mode.\n");
+				return out;
+			}
+			out[p + key] = format("%d", tenantMode);
+		}
 		return out;
 	}
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9169dd62f9..82be820771 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2859,6 +2859,7 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
 					         ssi.second,
 					         &StorageServerInterface::getValue,
 					         GetValueRequest(span.context,
+					                         TenantInfo(),
 					                         key,
 					                         ver,
 					                         trState->cx->sampleReadTags() ? trState->options.readTags
@@ -2964,6 +2965,7 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState, KeySelector k, Fut
 			++trState->cx->transactionPhysicalReads;
 
 			GetKeyRequest req(span.context,
+			                  TenantInfo(),
 			                  k,
 			                  version.get(),
 			                  trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>(),
@@ -3096,6 +3098,7 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 				                     ssi.second,
 				                     &StorageServerInterface::watchValue,
 				                     WatchValueRequest(span.context,
+				                                       TenantInfo(),
 				                                       parameters->key,
 				                                       parameters->value,
 				                                       ver,
@@ -4463,7 +4466,8 @@ Transaction::Transaction(Database const& cx)
                                             cx->taskID,
                                             generateSpanID(cx->transactionTracingSample),
                                             createTrLogInfoProbabilistically(cx))),
-    span(trState->spanID, "Transaction"_loc), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), tr(trState->spanID) {
+    span(trState->spanID, "Transaction"_loc), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF),
+    tr(TenantInfo(), trState->spanID) {
 	if (DatabaseContext::debugUseTags) {
 		debugAddTags(trState);
 	}
@@ -5092,7 +5096,7 @@ void TransactionOptions::reset(Database const& cx) {
 void Transaction::resetImpl(bool generateNewSpan) {
 	flushTrLogsIfEnabled();
 	trState = trState->cloneAndReset(createTrLogInfoProbabilistically(trState->cx), generateNewSpan);
-	tr = CommitTransactionRequest(trState->spanID);
+	tr = CommitTransactionRequest(TenantInfo(), trState->spanID);
 	readVersion = Future<Version>();
 	metadataVersion = Promise<Optional<Key>>();
 	extraConflictRanges.clear();
@@ -6663,7 +6667,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<Transa
 			for (int i = 0; i < nLocs; i++) {
 				partBegin = (i == 0) ? keys.begin : locations[i].first.begin;
 				partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end;
-				SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize);
+				SplitRangeRequest req(TenantInfo(), KeyRangeRef(partBegin, partEnd), chunkSize);
 				fReplies[i] = loadBalance(locations[i].second->locations(),
 				                          &StorageServerInterface::getRangeSplitPoints,
 				                          req,
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index c5e2f32792..3988202e93 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -809,6 +809,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
              "disabled",
              "aggressive",
              "gradual"
+         ]},
+         "tenant_mode": {
+             "$enum":[
+             "optional",
+             "required"
          ]}
       },
       "data":{
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 9e40f95e1f..17927c5d9d 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -33,6 +33,7 @@
 #include "fdbrpc/TSSComparison.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/Tenant.h"
 #include "flow/UnitTest.h"
 
 // Dead code, removed in the next protocol version
@@ -212,6 +213,19 @@ struct ServerCacheInfo {
 	}
 };
 
+struct TenantInfo {
+	Optional<TenantName> name;
+
+	TenantInfo() {}
+	TenantInfo(Optional<TenantName> name) : name(name) {}
+	TenantInfo(TenantName name) : name(name) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, name);
+	}
+};
+
 struct GetValueReply : public LoadBalancedReply {
 	constexpr static FileIdentifier file_identifier = 1378929;
 	Optional<Value> value;
@@ -229,6 +243,7 @@ struct GetValueReply : public LoadBalancedReply {
 struct GetValueRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 8454530;
 	SpanID spanContext;
+	TenantInfo tenantInfo;
 	Key key;
 	Version version;
 	Optional<TagSet> tags;
@@ -236,12 +251,17 @@ struct GetValueRequest : TimedRequest {
 	ReplyPromise<GetValueReply> reply;
 
 	GetValueRequest() {}
-	GetValueRequest(SpanID spanContext, const Key& key, Version ver, Optional<TagSet> tags, Optional<UID> debugID)
-	  : spanContext(spanContext), key(key), version(ver), tags(tags), debugID(debugID) {}
+	GetValueRequest(SpanID spanContext,
+	                const TenantInfo& tenantInfo,
+	                const Key& key,
+	                Version ver,
+	                Optional<TagSet> tags,
+	                Optional<UID> debugID)
+	  : spanContext(spanContext), tenantInfo(tenantInfo), key(key), version(ver), tags(tags), debugID(debugID) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, key, version, tags, debugID, reply, spanContext);
+		serializer(ar, key, version, tags, debugID, reply, spanContext, tenantInfo);
 	}
 };
 
@@ -262,6 +282,7 @@ struct WatchValueReply {
 struct WatchValueRequest {
 	constexpr static FileIdentifier file_identifier = 14747733;
 	SpanID spanContext;
+	TenantInfo tenantInfo;
 	Key key;
 	Optional<Value> value;
 	Version version;
@@ -270,17 +291,20 @@ struct WatchValueRequest {
 	ReplyPromise<WatchValueReply> reply;
 
 	WatchValueRequest() {}
+
 	WatchValueRequest(SpanID spanContext,
+	                  TenantInfo tenantInfo,
 	                  const Key& key,
 	                  Optional<Value> value,
 	                  Version ver,
 	                  Optional<TagSet> tags,
 	                  Optional<UID> debugID)
-	  : spanContext(spanContext), key(key), value(value), version(ver), tags(tags), debugID(debugID) {}
+	  : spanContext(spanContext), tenantInfo(tenantInfo), key(key), value(value), version(ver), tags(tags),
+	    debugID(debugID) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, key, value, version, tags, debugID, reply, spanContext);
+		serializer(ar, key, value, version, tags, debugID, reply, spanContext, tenantInfo);
 	}
 };
 
@@ -304,6 +328,7 @@ struct GetKeyValuesRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
 	SpanID spanContext;
 	Arena arena;
+	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
 	// This is a dummy field there has never been used.
 	// TODO: Get rid of this by constexpr or other template magic in getRange
@@ -316,9 +341,22 @@ struct GetKeyValuesRequest : TimedRequest {
 	ReplyPromise<GetKeyValuesReply> reply;
 
 	GetKeyValuesRequest() : isFetchKeys(false) {}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, begin, end, version, limit, limitBytes, isFetchKeys, tags, debugID, reply, spanContext, arena);
+		serializer(ar,
+		           begin,
+		           end,
+		           version,
+		           limit,
+		           limitBytes,
+		           isFetchKeys,
+		           tags,
+		           debugID,
+		           reply,
+		           spanContext,
+		           tenantInfo,
+		           arena);
 	}
 };
 
@@ -342,6 +380,7 @@ struct GetKeyValuesAndFlatMapRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 6795747;
 	SpanID spanContext;
 	Arena arena;
+	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
 	KeyRef mapper;
 	Version version; // or latestVersion
@@ -352,10 +391,23 @@ struct GetKeyValuesAndFlatMapRequest : TimedRequest {
 	ReplyPromise<GetKeyValuesAndFlatMapReply> reply;
 
 	GetKeyValuesAndFlatMapRequest() : isFetchKeys(false) {}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(
-		    ar, begin, end, mapper, version, limit, limitBytes, isFetchKeys, tags, debugID, reply, spanContext, arena);
+		serializer(ar,
+		           begin,
+		           end,
+		           mapper,
+		           version,
+		           limit,
+		           limitBytes,
+		           isFetchKeys,
+		           tags,
+		           debugID,
+		           reply,
+		           spanContext,
+		           tenantInfo,
+		           arena);
 	}
 };
 
@@ -390,6 +442,7 @@ struct GetKeyValuesStreamRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
 	SpanID spanContext;
 	Arena arena;
+	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
 	Version version; // or latestVersion
 	int limit, limitBytes;
@@ -399,9 +452,22 @@ struct GetKeyValuesStreamRequest {
 	ReplyPromiseStream<GetKeyValuesStreamReply> reply;
 
 	GetKeyValuesStreamRequest() : isFetchKeys(false) {}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, begin, end, version, limit, limitBytes, isFetchKeys, tags, debugID, reply, spanContext, arena);
+		serializer(ar,
+		           begin,
+		           end,
+		           version,
+		           limit,
+		           limitBytes,
+		           isFetchKeys,
+		           tags,
+		           debugID,
+		           reply,
+		           spanContext,
+		           tenantInfo,
+		           arena);
 	}
 };
 
@@ -423,6 +489,7 @@ struct GetKeyRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 10457870;
 	SpanID spanContext;
 	Arena arena;
+	TenantInfo tenantInfo;
 	KeySelectorRef sel;
 	Version version; // or latestVersion
 	Optional<TagSet> tags;
@@ -430,16 +497,18 @@ struct GetKeyRequest : TimedRequest {
 	ReplyPromise<GetKeyReply> reply;
 
 	GetKeyRequest() {}
+
 	GetKeyRequest(SpanID spanContext,
+	              TenantInfo tenantInfo,
 	              KeySelectorRef const& sel,
 	              Version version,
 	              Optional<TagSet> tags,
 	              Optional<UID> debugID)
-	  : spanContext(spanContext), sel(sel), version(version), debugID(debugID) {}
+	  : spanContext(spanContext), tenantInfo(tenantInfo), sel(sel), version(version), debugID(debugID) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, sel, version, tags, debugID, reply, spanContext, arena);
+		serializer(ar, sel, version, tags, debugID, reply, spanContext, tenantInfo, arena);
 	}
 };
 
@@ -671,16 +740,18 @@ struct SplitRangeReply {
 struct SplitRangeRequest {
 	constexpr static FileIdentifier file_identifier = 10725174;
 	Arena arena;
+	TenantInfo tenantInfo;
 	KeyRangeRef keys;
 	int64_t chunkSize;
 	ReplyPromise<SplitRangeReply> reply;
 
 	SplitRangeRequest() {}
-	SplitRangeRequest(KeyRangeRef const& keys, int64_t chunkSize) : keys(arena, keys), chunkSize(chunkSize) {}
+	SplitRangeRequest(TenantInfo tenantInfo, KeyRangeRef const& keys, int64_t chunkSize)
+	  : tenantInfo(tenantInfo), keys(arena, keys), chunkSize(chunkSize) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, keys, chunkSize, reply, arena);
+		serializer(ar, keys, chunkSize, reply, tenantInfo, arena);
 	}
 };
 
@@ -711,6 +782,7 @@ struct ChangeFeedStreamRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
 	SpanID spanContext;
 	Arena arena;
+	TenantInfo tenantInfo;
 	Key rangeID;
 	Version begin = 0;
 	Version end = 0;
@@ -720,7 +792,7 @@ struct ChangeFeedStreamRequest {
 	ChangeFeedStreamRequest() {}
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeID, begin, end, range, reply, spanContext, arena);
+		serializer(ar, rangeID, begin, end, range, reply, spanContext, tenantInfo, arena);
 	}
 };
 
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index fe10cae868..82c194f5db 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -1322,6 +1322,21 @@ BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value) {
 	return interf;
 }
 
+Value encodeTenantEntry(TenantMapEntry const& tenantEntry) {
+	return ObjectWriter::toValue(tenantEntry, IncludeVersion());
+}
+
+TenantMapEntry decodeTenantEntry(ValueRef const& value) {
+	TenantMapEntry entry;
+	ObjectReader reader(value.begin(), IncludeVersion());
+	reader.deserialize(entry);
+	return entry;
+}
+
+const KeyRangeRef tenantMapKeys("\xff/tenantMap/"_sr, "\xff/tenantMap0"_sr);
+const KeyRef tenantMapPrefix = tenantMapKeys.begin;
+const KeyRef tenantMapPrivatePrefix = "\xff\xff/tenantMap/"_sr;
+
 // for tests
 void testSSISerdes(StorageServerInterface const& ssi, bool useFB) {
 	printf("ssi=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\naddress=%s\ngetValue=%s\n\n\n",
diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h
index 14234151a3..993efb1dba 100644
--- a/fdbclient/SystemData.h
+++ b/fdbclient/SystemData.h
@@ -27,6 +27,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BlobWorkerInterface.h" // TODO move the functions that depend on this out of here and into BlobWorkerInterface.h to remove this depdendency
 #include "fdbclient/StorageServerInterface.h"
+#include "Tenant.h"
 
 // Don't warn on constants being defined in this file.
 #pragma clang diagnostic push
@@ -593,6 +594,14 @@ UID decodeBlobWorkerListKey(KeyRef const& key);
 const Value blobWorkerListValue(BlobWorkerInterface const& interface);
 BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value);
 
+// State for the tenant map
+extern const KeyRangeRef tenantMapKeys;
+extern const KeyRef tenantMapPrefix;
+extern const KeyRef tenantMapPrivatePrefix;
+
+Value encodeTenantEntry(TenantMapEntry const& tenantEntry);
+TenantMapEntry decodeTenantEntry(ValueRef const& value);
+
 #pragma clang diagnostic pop
 
 #endif
diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
new file mode 100644
index 0000000000..bf6cc0f1bc
--- /dev/null
+++ b/fdbclient/Tenant.h
@@ -0,0 +1,49 @@
+/*
+ * Tenant.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_TENANT_H
+#define FDBCLIENT_TENANT_H
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/VersionedMap.h"
+#include "flow/flat_buffers.h"
+
+typedef StringRef TenantNameRef;
+typedef Standalone<TenantNameRef> TenantName;
+
+struct TenantMapEntry {
+	constexpr static FileIdentifier file_identifier = 12247338;
+
+	Key prefix;
+
+	TenantMapEntry() {}
+	TenantMapEntry(Key prefix) : prefix(prefix) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, prefix);
+	}
+};
+
+typedef VersionedMap<TenantName, TenantMapEntry> TenantMap;
+typedef VersionedMap<KeyRef, TenantNameRef> TenantPrefixIndex;
+
+#endif
\ No newline at end of file
diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp
index 47871915f9..e90080dca3 100644
--- a/fdbserver/ApplyMetadataMutation.cpp
+++ b/fdbserver/ApplyMetadataMutation.cpp
@@ -70,16 +70,19 @@ public:
 	                           Reference<ILogSystem> logSystem_,
 	                           LogPushData* toCommit_,
 	                           bool& confChange_,
+	                           Version version,
 	                           Version popVersion_,
 	                           bool initialCommit_)
 	  : spanContext(spanContext_), dbgid(proxyCommitData_.dbgid), arena(arena_), mutations(mutations_),
 	    txnStateStore(proxyCommitData_.txnStateStore), toCommit(toCommit_), confChange(confChange_),
-	    logSystem(logSystem_), popVersion(popVersion_), vecBackupKeys(&proxyCommitData_.vecBackupKeys),
-	    keyInfo(&proxyCommitData_.keyInfo), cacheInfo(&proxyCommitData_.cacheInfo),
+	    logSystem(logSystem_), version(version), popVersion(popVersion_),
+	    vecBackupKeys(&proxyCommitData_.vecBackupKeys), keyInfo(&proxyCommitData_.keyInfo),
+	    cacheInfo(&proxyCommitData_.cacheInfo),
 	    uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr),
-	    commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), commitVersion(&proxyCommitData_.committedVersion),
+	    commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion),
 	    storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped),
-	    tssMapping(&proxyCommitData_.tssMapping), initialCommit(initialCommit_) {}
+	    tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap),
+	    initialCommit(initialCommit_) {}
 
 private:
 	// The following variables are incoming parameters
@@ -102,6 +105,7 @@ private:
 	bool& confChange;
 
 	Reference<ILogSystem> logSystem = Reference<ILogSystem>();
+	Version version = invalidVersion;
 	Version popVersion = 0;
 	KeyRangeMap<std::set<Key>>* vecBackupKeys = nullptr;
 	KeyRangeMap<ServerCacheInfo>* keyInfo = nullptr;
@@ -109,11 +113,13 @@ private:
 	std::map<Key, ApplyMutationsData>* uid_applyMutationsData = nullptr;
 	RequestStream<CommitTransactionRequest> commit = RequestStream<CommitTransactionRequest>();
 	Database cx = Database();
-	NotifiedVersion* commitVersion = nullptr;
+	NotifiedVersion* committedVersion = nullptr;
 	std::map<UID, Reference<StorageInfo>>* storageCache = nullptr;
 	std::map<Tag, Version>* tag_popped = nullptr;
 	std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
 
+	TenantMap* tenantMap = nullptr;
+
 	// true if the mutations were already written to the txnStateStore as part of recovery
 	bool initialCommit = false;
 
@@ -445,7 +451,7 @@ private:
 		    beginValue.present() ? BinaryReader::fromStringRef<Version>(beginValue.get(), Unversioned()) : 0,
 		    &p.endVersion,
 		    commit,
-		    commitVersion,
+		    committedVersion,
 		    p.keyVersion);
 	}
 
@@ -571,6 +577,41 @@ private:
 		TEST(true); // Snapshot created, setting writeRecoveryKey in txnStateStore
 	}
 
+	void checkSetTenantMapPrefix(MutationRef m) {
+		if (m.param1.startsWith(tenantMapPrefix)) {
+			if (tenantMap) {
+				ASSERT(version != invalidVersion);
+				tenantMap->createNewVersion(version);
+				Standalone<StringRef> tenantName = m.param1.removePrefix(tenantMapPrefix);
+				TenantMapEntry tenantEntry = decodeTenantEntry(m.param2);
+
+				tenantMap->insert(tenantName, tenantEntry);
+			}
+
+			if (!initialCommit) {
+				txnStateStore->set(KeyValueRef(m.param1, m.param2));
+			}
+
+			// For now, this goes to all storage servers.
+			// Eventually, we can have each SS store tenants that apply only to the data stored on it.
+			if (toCommit) {
+				std::set<Tag> allTags;
+				auto allServers = txnStateStore->readRange(serverTagKeys).get();
+				for (auto& kv : allServers) {
+					allTags.insert(decodeServerTagValue(kv.value));
+				}
+
+				toCommit->addTags(allTags);
+
+				MutationRef privatized = m;
+				privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena);
+				toCommit->writeTypedMessage(privatized);
+			}
+
+			TEST(true); // Tenant added to map
+		}
+	}
+
 	void checkClearKeyServerKeys(KeyRangeRef range) {
 		if (!keyServersKeys.intersects(range)) {
 			return;
@@ -893,6 +934,44 @@ private:
 		}
 	}
 
+	void checkClearTenantMapPrefix(KeyRangeRef range) {
+		if (tenantMapKeys.intersects(range)) {
+			if (tenantMap) {
+				ASSERT(version != invalidVersion);
+				tenantMap->createNewVersion(version);
+
+				StringRef startTenant = std::max(range.begin, tenantMapPrefix).removePrefix(tenantMapPrefix);
+				StringRef endTenant = range.end.startsWith(tenantMapPrefix) ? range.end : tenantMapKeys.end;
+
+				tenantMap->erase(startTenant, endTenant);
+			}
+
+			if (!initialCommit) {
+				txnStateStore->clear(range);
+			}
+
+			// For now, this goes to all storage servers.
+			// Eventually, we can have each SS store tenants that apply only to the data stored on it.
+			if (toCommit) {
+				std::set<Tag> allTags;
+				auto allServers = txnStateStore->readRange(serverTagKeys).get();
+				for (auto& kv : allServers) {
+					allTags.insert(decodeServerTagValue(kv.value));
+				}
+
+				toCommit->addTags(allTags);
+
+				MutationRef privatized;
+				privatized.type = MutationRef::ClearRange;
+				privatized.param1 = range.begin.withPrefix(systemKeys.begin, arena);
+				privatized.param2 = range.end.withPrefix(systemKeys.begin, arena);
+				toCommit->writeTypedMessage(privatized);
+			}
+
+			TEST(true); // Tenant cleared from map
+		}
+	}
+
 	void checkClearMiscRangeKeys(KeyRangeRef range) {
 		if (initialCommit) {
 			return;
@@ -1009,6 +1088,7 @@ public:
 				checkSetGlobalKeys(m);
 				checkSetWriteRecoverKey(m);
 				checkSetMinRequiredCommitVersionKey(m);
+				checkSetTenantMapPrefix(m);
 				checkSetOtherKeys(m);
 			} else if (m.type == MutationRef::ClearRange && isSystemKey(m.param2)) {
 				KeyRangeRef range(m.param1, m.param2);
@@ -1024,6 +1104,7 @@ public:
 				checkClearLogRangesRange(range);
 				checkClearTssMappingKeys(m, range);
 				checkClearTssQuarantineKeys(m, range);
+				checkClearTenantMapPrefix(range);
 				checkClearMiscRangeKeys(range);
 			}
 		}
@@ -1052,11 +1133,20 @@ void applyMetadataMutations(SpanID const& spanContext,
                             const VectorRef<MutationRef>& mutations,
                             LogPushData* toCommit,
                             bool& confChange,
+                            Version version,
                             Version popVersion,
                             bool initialCommit) {
 
-	ApplyMetadataMutationsImpl(
-	    spanContext, arena, mutations, proxyCommitData, logSystem, toCommit, confChange, popVersion, initialCommit)
+	ApplyMetadataMutationsImpl(spanContext,
+	                           arena,
+	                           mutations,
+	                           proxyCommitData,
+	                           logSystem,
+	                           toCommit,
+	                           confChange,
+	                           version,
+	                           popVersion,
+	                           initialCommit)
 	    .apply();
 }
 
diff --git a/fdbserver/ApplyMetadataMutation.h b/fdbserver/ApplyMetadataMutation.h
index f3cc17e07d..bdceaf8d8e 100644
--- a/fdbserver/ApplyMetadataMutation.h
+++ b/fdbserver/ApplyMetadataMutation.h
@@ -56,6 +56,7 @@ void applyMetadataMutations(SpanID const& spanContext,
                             const VectorRef<MutationRef>& mutations,
                             LogPushData* pToCommit,
                             bool& confChange,
+                            Version version,
                             Version popVersion,
                             bool initialCommit);
 void applyMetadataMutations(SpanID const& spanContext,
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 5860e9bd84..753ec0e2ef 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -226,6 +226,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig;
 			dbInfo.myLocality = db->serverInfo->get().myLocality;
 			dbInfo.client = ClientDBInfo();
+			dbInfo.client.tenantMode = db->config.tenantMode;
 
 			TraceEvent("CCWDB", cluster->id)
 			    .detail("NewMaster", dbInfo.master.id().toString())
@@ -980,19 +981,23 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 
 	// Construct the client information
 	if (db->clientInfo->get().commitProxies != req.commitProxies ||
-	    db->clientInfo->get().grvProxies != req.grvProxies) {
+	    db->clientInfo->get().grvProxies != req.grvProxies ||
+	    db->clientInfo->get().tenantMode != db->config.tenantMode) {
 		TraceEvent("PublishNewClientInfo", self->id)
 		    .detail("Master", dbInfo.master.id())
 		    .detail("GrvProxies", db->clientInfo->get().grvProxies)
 		    .detail("ReqGrvProxies", req.grvProxies)
 		    .detail("CommitProxies", db->clientInfo->get().commitProxies)
-		    .detail("ReqCPs", req.commitProxies);
+		    .detail("ReqCPs", req.commitProxies)
+		    .detail("TenantMode", db->clientInfo->get().tenantMode.toString())
+		    .detail("ReqTenantMode", db->config.tenantMode.toString());
 		isChanged = true;
 		// TODO why construct a new one and not just copy the old one and change proxies + id?
 		ClientDBInfo clientInfo;
 		clientInfo.id = deterministicRandom()->randomUniqueID();
 		clientInfo.commitProxies = req.commitProxies;
 		clientInfo.grvProxies = req.grvProxies;
+		clientInfo.tenantMode = db->config.tenantMode;
 		db->clientInfo->set(clientInfo);
 		dbInfo.client = db->clientInfo->get();
 	}
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 17429f6cea..f474b00f85 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -762,6 +762,7 @@ void applyMetadataEffect(CommitBatchContext* self) {
 				                       self->resolution[0].stateMutations[versionIndex][transactionIndex].mutations,
 				                       /* pToCommit= */ nullptr,
 				                       self->forceRecovery,
+				                       /* version= */ self->commitVersion,
 				                       /* popVersion= */ 0,
 				                       /* initialCommit */ false);
 			}
@@ -836,6 +837,33 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 	}
 }
 
+ErrorOr<Void> checkTenant(ProxyCommitData* commitData, CommitTransactionRequest const& tr, Version commitVersion) {
+	if (tr.tenantInfo.name.present()) {
+		auto view = commitData->tenantMap.at(commitVersion);
+		auto itr = view.find(tr.tenantInfo.name.get());
+		if (itr == view.end()) {
+			TraceEvent(SevWarn, "CommitProxyTenantNotFound", commitData->dbgid)
+			    .detail("Tenant", tr.tenantInfo.name.get());
+			return tenant_not_found();
+		}
+
+		for (auto mutation : tr.transaction.mutations) {
+			if (!mutation.param1.startsWith(itr->prefix) ||
+			    (mutation.type == MutationRef::ClearRange && !mutation.param2.startsWith(itr->prefix))) {
+				TraceEvent(SevWarn, "KeyNotInTenant", commitData->dbgid)
+				    .detail("Tenant", tr.tenantInfo.name.get())
+				    .detail("TenantPrefix", itr->prefix)
+				    .detail("Key", mutation.param1)
+				    .detail("EndKey",
+				            mutation.type == MutationRef::ClearRange ? mutation.param2 : Optional<StringRef>());
+				throw key_not_in_tenant();
+			}
+		}
+	}
+
+	return Void();
+}
+
 // This first pass through committed transactions deals with "metadata" effects (modifications of txnStateStore, changes
 // to storage servers' responsibilities)
 ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self) {
@@ -845,16 +873,24 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	int t;
 	for (t = 0; t < trs.size() && !self->forceRecovery; t++) {
 		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) {
-			self->commitCount++;
-			applyMetadataMutations(trs[t].spanContext,
-			                       *pProxyCommitData,
-			                       self->arena,
-			                       pProxyCommitData->logSystem,
-			                       trs[t].transaction.mutations,
-			                       &self->toCommit,
-			                       self->forceRecovery,
-			                       self->commitVersion + 1,
-			                       /* initialCommit= */ false);
+			ErrorOr<Void> result = checkTenant(pProxyCommitData, trs[t], self->commitVersion);
+
+			if (result.isError()) {
+				self->committed[t] = ConflictBatch::TransactionTenantFailure;
+				trs[t].reply.sendError(result.getError());
+			} else {
+				self->commitCount++;
+				applyMetadataMutations(trs[t].spanContext,
+				                       *pProxyCommitData,
+				                       self->arena,
+				                       pProxyCommitData->logSystem,
+				                       trs[t].transaction.mutations,
+				                       &self->toCommit,
+				                       self->forceRecovery,
+				                       self->commitVersion,
+				                       self->commitVersion + 1,
+				                       /* initialCommit= */ false);
+			}
 		}
 		if (self->firstStateMutations) {
 			ASSERT(self->committed[t] == ConflictBatch::TransactionCommitted);
@@ -1346,6 +1382,9 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 			tr.reply.send(CommitID(self->commitVersion, t, self->metadataVersionAfter));
 		} else if (self->committed[t] == ConflictBatch::TransactionTooOld) {
 			tr.reply.sendError(transaction_too_old());
+		} else if (self->committed[t] == ConflictBatch::TransactionTenantFailure) {
+			// We already sent the error
+			ASSERT(tr.reply.isSet());
 		} else {
 			// If enable the option to report conflicting keys from resolvers, we send back all keyranges' indices
 			// through CommitID
@@ -1958,6 +1997,7 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 		                       mutations,
 		                       /* pToCommit= */ nullptr,
 		                       confChanges,
+		                       /* version= */ 0,
 		                       /* popVersion= */ 0,
 		                       /* initialCommit= */ true);
 	} // loop
diff --git a/fdbserver/ConflictSet.h b/fdbserver/ConflictSet.h
index 0b6ea47f87..f68e2d8aac 100644
--- a/fdbserver/ConflictSet.h
+++ b/fdbserver/ConflictSet.h
@@ -41,6 +41,7 @@ struct ConflictBatch {
 	enum TransactionCommitResult {
 		TransactionConflict = 0,
 		TransactionTooOld,
+		TransactionTenantFailure,
 		TransactionCommitted,
 	};
 
diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h
index 7ca822cfb6..249a7c7a60 100644
--- a/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/ProxyCommitData.actor.h
@@ -26,6 +26,7 @@
 #define FDBSERVER_PROXYCOMMITDATA_ACTOR_H
 
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/Tenant.h"
 #include "fdbrpc/Stats.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogSystemDiskQueueAdapter.h"
@@ -214,6 +215,8 @@ struct ProxyCommitData {
 	double lastMasterReset;
 	double lastResolverReset;
 
+	TenantMap tenantMap;
+
 	// The tag related to a storage server rarely change, so we keep a vector of tags for each key range to be slightly
 	// more CPU efficient. When a tag related to a storage server does change, we empty out all of these vectors to
 	// signify they must be repopulated. We do not repopulate them immediately to avoid a slow task.
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index e7cb21e74a..4408fd500b 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -1811,7 +1811,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		if (kv.second.type() == json_spirit::int_type) {
 			startingConfigString += kv.first + ":=" + format("%d", kv.second.get_int());
 		} else if (kv.second.type() == json_spirit::str_type) {
-			if ("storage_migration_type" == kv.first) {
+			if ("storage_migration_type" == kv.first || "tenant_mode" == kv.first) {
 				startingConfigString += kv.first + "=" + kv.second.get_str();
 			} else {
 				startingConfigString += kv.second.get_str();
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index a4fed0c584..d26bbb45b1 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -42,6 +42,7 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/StatusClient.h"
+#include "fdbclient/Tenant.h"
 #include "fdbclient/Tuple.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TransactionLineage.h"
@@ -85,6 +86,9 @@ bool canReplyWith(Error e) {
 	case error_code_watch_cancelled:
 	case error_code_unknown_change_feed:
 	case error_code_server_overloaded:
+	case error_code_tenant_name_required:
+	case error_code_tenant_not_found:
+	case error_code_key_not_in_tenant:
 	// getRangeAndMap related exceptions that are not retriable:
 	case error_code_mapper_bad_index:
 	case error_code_mapper_no_such_key:
@@ -414,7 +418,9 @@ private:
 	std::unordered_map<KeyRef, Reference<ServerWatchMetadata>> watchMap; // keep track of server watches
 
 public:
-public:
+	TenantMap tenantMap;
+	TenantPrefixIndex tenantPrefixIndex;
+
 	// Histograms
 	struct FetchKeysHistograms {
 		const Reference<Histogram> latency;
@@ -448,6 +454,16 @@ public:
 	void deleteWatchMetadata(KeyRef key);
 	void clearWatchMetadata();
 
+	// tenant map operations
+	void insertTenant(TenantNameRef tenantName, ValueRef value, Version version, bool insertIntoMutationLog);
+	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
+
+	Optional<TenantMapEntry> checkTenant(Version version,
+	                                     Optional<TenantName> tenant,
+	                                     KeyRef begin,
+	                                     Optional<KeyRef> end);
+	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantName> tenant, Version version, Arena& arena);
+
 	class CurrentRunningFetchKeys {
 		std::unordered_map<UID, double> startTimeMap;
 		std::unordered_map<UID, KeyRange> keyRangeMap;
@@ -1123,6 +1139,15 @@ public:
 		}
 		return fun(this, request);
 	}
+
+	void getSplitPoints(SplitRangeRequest const& req) {
+		try {
+			checkTenant(version.get(), req.tenantInfo.name, req.keys.begin, req.keys.end);
+			metrics.getSplitPoints(req);
+		} catch (Error& e) {
+			req.reply.sendError(e);
+		}
+	}
 };
 
 const StringRef StorageServer::CurrentRunningFetchKeys::emptyString = LiteralStringRef("");
@@ -1351,9 +1376,44 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageServer* data, Version versio
 	}
 }
 
+Optional<TenantMapEntry> StorageServer::checkTenant(Version version,
+                                                    Optional<TenantName> tenantName,
+                                                    KeyRef begin,
+                                                    Optional<KeyRef> end = Optional<KeyRef>()) {
+	if (tenantName.present()) {
+		auto view = tenantMap.at(version);
+		auto itr = view.find(tenantName.get());
+		if (itr == view.end()) {
+			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID)
+			    .detail("Tenant", tenantName)
+			    .detail("Begin", begin)
+			    .detail("End", end)
+			    .backtrace();
+			throw tenant_not_found();
+		}
+
+		if (!begin.startsWith(itr->prefix) || (end.present() && !end.get().startsWith(itr->prefix))) {
+			TraceEvent(SevWarn, "KeyNotInTenant", thisServerID)
+			    .detail("Tenant", tenantName)
+			    .detail("TenantPrefix", itr->prefix)
+			    .detail("Begin", begin)
+			    .detail("End", end)
+			    .backtrace();
+			throw key_not_in_tenant();
+		}
+
+		return *itr;
+	}
+
+	return Optional<TenantMapEntry>();
+}
+
 ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 	state int64_t resultSize = 0;
 	Span span("SS:getValue"_loc, { req.spanContext });
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
 	span.addTag("key"_sr, req.key);
 	// Temporarily disabled -- this path is hit a lot
 	// getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
@@ -1381,6 +1441,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			                      req.debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
+		data->checkTenant(version, req.tenantInfo.name, req.key);
 		state uint64_t changeCounter = data->shardChangeCounter;
 
 		if (!data->shards[req.key]->isReadable()) {
@@ -1506,7 +1567,8 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanID parent
 			state Version latest = data->version.get();
 			TEST(latest >= minVersion &&
 			     latest < data->data().latestVersion); // Starting watch loop with latestVersion > data->version
-			GetValueRequest getReq(span.context, metadata->key, latest, metadata->tags, metadata->debugID);
+			GetValueRequest getReq(
+			    span.context, TenantInfo(), metadata->key, latest, metadata->tags, metadata->debugID);
 			state Future<Void> getValue = getValueQ(
 			    data, getReq); // we are relying on the delay zero at the top of getValueQ, if removed we need one here
 			GetValueReply reply = wait(getReq.reply.getFuture());
@@ -1801,6 +1863,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		throw wrong_shard_server();
 	}
 
+	data->checkTenant(req.begin, req.tenantInfo.name, req.range.begin, req.range.end);
+
 	auto feed = data->uidChangeFeed.find(req.rangeID);
 	if (feed == data->uidChangeFeed.end()) {
 		throw unknown_change_feed();
@@ -2141,7 +2205,8 @@ ACTOR Future<Optional<Value>> quickGetValue(StorageServer* data,
 	if (data->shards[key]->isReadable()) {
 		try {
 			// TODO: Use a lower level API may be better? Or tweak priorities?
-			GetValueRequest req(pOriginalReq->spanContext, key, version, pOriginalReq->tags, pOriginalReq->debugID);
+			GetValueRequest req(
+			    pOriginalReq->spanContext, TenantInfo(), key, version, pOriginalReq->tags, pOriginalReq->debugID);
 			// Note that it does not use readGuard to avoid server being overloaded here. Throttling is enforced at the
 			// original request level, rather than individual underlying lookups. The reason is that throttle any
 			// individual underlying lookup will fail the original request, which is not productive.
@@ -2375,10 +2440,21 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 	return result;
 }
 
-// bool selectorInRange( KeySelectorRef const& sel, KeyRangeRef const& range ) {
-// Returns true if the given range suffices to at least begin to resolve the given KeySelectorRef
-//	return sel.getKey() >= range.begin && (sel.isBackward() ? sel.getKey() <= range.end : sel.getKey() < range.end);
-//}
+KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range,
+                                              Optional<TenantName> tenant,
+                                              Version version,
+                                              Arena& arena) {
+	if (tenant.present()) {
+		auto view = tenantMap.at(version);
+		auto itr = view.find(tenant.get());
+		ASSERT(itr != view.end());
+
+		return KeyRangeRef(range.begin.startsWith(itr->prefix) ? range.begin : itr->prefix,
+		                   range.end.startsWith(itr->prefix) ? range.end : allKeys.end.withPrefix(itr->prefix, arena));
+	} else {
+		return range;
+	}
+}
 
 ACTOR Future<Key> findKey(StorageServer* data,
                           KeySelectorRef sel,
@@ -2501,6 +2577,11 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 	state int64_t resultSize = 0;
 	state IKeyValueStore::ReadType type =
 	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
+
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
+
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 
 	++data->counters.getRangeQueries;
@@ -2522,6 +2603,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
+		data->checkTenant(version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
@@ -2541,14 +2623,16 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			throw wrong_shard_server();
 		}
 
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+
 		state int offset1 = 0;
 		state int offset2;
 		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
 		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, type);
+		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 
@@ -2706,7 +2790,10 @@ ACTOR Future<RangeResult> quickGetKeyValues(StorageServer* data,
 	}
 };
 
-Key constructMappedKey(KeyValueRef* keyValue, Tuple& mappedKeyFormatTuple, bool& isRangeQuery) {
+Key constructMappedKey(KeyValueRef* keyValue,
+                       Tuple& mappedKeyFormatTuple,
+                       bool& isRangeQuery,
+                       Optional<TenantMapEntry> tenantEntry) {
 	// Lazily parse key and/or value to tuple because they may not need to be a tuple if not used.
 	Optional<Tuple> keyTuple;
 	Optional<Tuple> valueTuple;
@@ -2795,7 +2882,14 @@ Key constructMappedKey(KeyValueRef* keyValue, Tuple& mappedKeyFormatTuple, bool&
 			mappedKeyTuple.append(mappedKeyFormatTuple.subTuple(i, i + 1));
 		}
 	}
-	return mappedKeyTuple.getDataAsStandalone();
+
+	KeyRef mappedKey = mappedKeyTuple.pack();
+
+	if (tenantEntry.present() && !tenantEntry.get().prefix.empty()) {
+		return mappedKey.withPrefix(tenantEntry.get().prefix);
+	}
+
+	return mappedKey;
 }
 
 TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
@@ -2811,7 +2905,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -2823,11 +2917,33 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == true);
 	}
+	{
+		Tuple mapperTuple = Tuple()
+		                        .append("normal"_sr)
+		                        .append("{{escaped}}"_sr)
+		                        .append("{K[2]}"_sr)
+		                        .append("{V[0]}"_sr)
+		                        .append("{...}"_sr);
+
+		bool isRangeQuery = false;
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry("Prefix"_sr));
+
+		Key expectedMappedKey = Tuple()
+		                            .append("normal"_sr)
+		                            .append("{escaped}"_sr)
+		                            .append("key-2"_sr)
+		                            .append("value-0"_sr)
+		                            .getDataAsStandalone()
+		                            .withPrefix("Prefix"_sr);
+		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
+		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
+		ASSERT(isRangeQuery == true);
+	}
 	{
 		Tuple mapperTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 
 		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
@@ -2838,7 +2954,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		Tuple mapperTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 
 		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
@@ -2850,7 +2966,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException = true;
@@ -2862,7 +2978,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException2 = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_range_decriptor);
 			throwException2 = true;
@@ -2874,7 +2990,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException3 = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery);
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException3 = true;
@@ -2888,7 +3004,8 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
                                                   GetKeyValuesReply input,
                                                   StringRef mapper,
                                                   // To provide span context, tags, debug ID to underlying lookups.
-                                                  GetKeyValuesAndFlatMapRequest* pOriginalReq) {
+                                                  GetKeyValuesAndFlatMapRequest* pOriginalReq,
+                                                  Optional<TenantMapEntry> tenantEntry) {
 	state GetKeyValuesAndFlatMapReply result;
 	result.version = input.version;
 	if (input.more) {
@@ -2903,9 +3020,7 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
 	state Tuple mappedKeyFormatTuple = Tuple::unpack(mapper);
 	state KeyValueRef* it = input.data.begin();
 	for (; it != input.data.end(); it++) {
-		state StringRef key = it->key;
-
-		state Key mappedKey = constructMappedKey(it, mappedKeyFormatTuple, isRangeQuery);
+		state Key mappedKey = constructMappedKey(it, mappedKeyFormatTuple, isRangeQuery, tenantEntry);
 		// Make sure the mappedKey is always available, so that it's good even we want to get key asynchronously.
 		result.arena.dependsOn(mappedKey.arena());
 
@@ -2946,6 +3061,11 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 	state int64_t resultSize = 0;
 	state IKeyValueStore::ReadType type =
 	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
+
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
+
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 
 	++data->counters.getRangeAndFlatMapQueries;
@@ -2968,6 +3088,9 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesAndFlatMap.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
+		state Optional<TenantMapEntry> tenantEntry =
+		    data->checkTenant(req.version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
+
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
@@ -2987,17 +3110,38 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			throw wrong_shard_server();
 		}
 
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+
 		state int offset1 = 0;
 		state int offset2;
 		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
 		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, type);
+		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 
+		if (!tenantEntry.present()) {
+			// We do not support raw flat map requests when using tenants.
+			// When tenants are required, we disable raw flat map requests entirely.
+			// If tenants are optional, we check whether the range intersects a tenant and fail if it does.
+			if (data->db->get().client.tenantMode == TenantMode::REQUIRED) {
+				throw tenant_name_required();
+			}
+
+			auto view = data->tenantPrefixIndex.at(req.version);
+			auto beginItr = view.lastLessOrEqual(begin);
+			if (beginItr != view.end() && !begin.startsWith(beginItr.key())) {
+				++beginItr;
+			}
+			auto endItr = view.lastLessOrEqual(end);
+			if (beginItr != endItr) {
+				throw tenant_name_required();
+			}
+		}
+
 		if (req.debugID.present())
 			g_traceBatch.addEvent(
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesAndFlatMap.AfterKeys");
@@ -3041,7 +3185,7 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			state GetKeyValuesAndFlatMapReply r;
 			try {
 				// Map the scanned range to another list of keys and look up.
-				GetKeyValuesAndFlatMapReply _r = wait(flatMap(data, getKeyValuesReply, req.mapper, &req));
+				GetKeyValuesAndFlatMapReply _r = wait(flatMap(data, getKeyValuesReply, req.mapper, &req, tenantEntry));
 				r = _r;
 			} catch (Error& e) {
 				TraceEvent("FlatMapError").error(e);
@@ -3128,6 +3272,10 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 	state IKeyValueStore::ReadType type =
 	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
 
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
+
 	req.reply.setByteLimit(SERVER_KNOBS->RANGESTREAM_LIMIT_BYTES);
 	++data->counters.getRangeStreamQueries;
 	++data->counters.allQueries;
@@ -3149,6 +3297,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
+		data->checkTenant(req.version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
@@ -3168,14 +3317,16 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			throw wrong_shard_server();
 		}
 
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+
 		state int offset1 = 0;
 		state int offset2;
 		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
 		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, type);
+		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 		if (req.debugID.present())
@@ -3308,6 +3459,9 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	state Span span("SS:getKey"_loc, { req.spanContext });
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
 	state int64_t resultSize = 0;
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 
@@ -3324,12 +3478,15 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	try {
 		state Version version = wait(waitForVersion(data, req.version, req.spanContext));
 
+		data->checkTenant(version, req.tenantInfo.name, req.sel.getKey());
 		state uint64_t changeCounter = data->shardChangeCounter;
-		state KeyRange shard = getShardKeyRange(data, req.sel);
+
+		KeyRange shard = getShardKeyRange(data, req.sel);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, req.version, req.arena);
 
 		state int offset;
-		Key k =
-		    wait(findKey(data, req.sel, version, shard, &offset, req.spanContext, IKeyValueStore::ReadType::NORMAL));
+		Key k = wait(
+		    findKey(data, req.sel, version, searchRange, &offset, req.spanContext, IKeyValueStore::ReadType::NORMAL));
 
 		data->checkChangeCounter(
 		    changeCounter, KeyRangeRef(std::min<KeyRef>(req.sel.getKey(), k), std::max<KeyRef>(req.sel.getKey(), k)));
@@ -3950,6 +4107,8 @@ static const KeyRef persistLogProtocol = LiteralStringRef(PERSIST_PREFIX "LogPro
 static const KeyRef persistPrimaryLocality = LiteralStringRef(PERSIST_PREFIX "PrimaryLocality");
 static const KeyRangeRef persistChangeFeedKeys =
     KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "RF/"), LiteralStringRef(PERSIST_PREFIX "RF0"));
+static const KeyRangeRef persistTenantMapKeys =
+    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "TM/"), LiteralStringRef(PERSIST_PREFIX "TM0"));
 // data keys are unmangled (but never start with PERSIST_PREFIX because they are always in allKeys)
 
 ACTOR Future<Void> fetchChangeFeedApplier(StorageServer* data,
@@ -5023,6 +5182,15 @@ private:
 					                m.param2));
 				}
 			}
+		} else if ((m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) &&
+		           m.param1.startsWith(tenantMapPrivatePrefix)) {
+			if (m.type == MutationRef::SetValue) {
+				data->insertTenant(m.param1.removePrefix(tenantMapPrivatePrefix), m.param2, currentVersion, true);
+			} else if (m.type == MutationRef::ClearRange) {
+				data->clearTenants(m.param1.removePrefix(tenantMapPrivatePrefix),
+				                   m.param2.removePrefix(tenantMapPrivatePrefix),
+				                   currentVersion);
+			}
 		} else if (m.param1.substr(1).startsWith(tssMappingKeys.begin) &&
 		           (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange)) {
 			if (!data->isTss()) {
@@ -5089,6 +5257,45 @@ private:
 	}
 };
 
+void StorageServer::insertTenant(TenantNameRef tenantName,
+                                 ValueRef value,
+                                 Version version,
+                                 bool insertIntoMutationLog) {
+	tenantMap.createNewVersion(version);
+	tenantPrefixIndex.createNewVersion(version);
+
+	TenantMapEntry tenantEntry = decodeTenantEntry(value);
+
+	tenantMap.insert(tenantName, tenantEntry);
+	tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
+
+	if (insertIntoMutationLog) {
+		auto& mLV = addVersionToMutationLog(version);
+		addMutationToMutationLog(
+		    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
+	}
+}
+
+void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version) {
+	tenantMap.createNewVersion(version);
+	tenantPrefixIndex.createNewVersion(version);
+
+	auto view = tenantMap.at(version);
+	for (auto itr = view.lower_bound(startTenant); itr != view.lower_bound(endTenant); ++itr) {
+		// Trigger any watches on the prefix associated with the tenant.
+		watches.triggerRange(itr->prefix, strinc(itr->prefix));
+		tenantPrefixIndex.erase(itr->prefix);
+	}
+
+	tenantMap.erase(startTenant, endTenant);
+
+	auto& mLV = addVersionToMutationLog(version);
+	addMutationToMutationLog(mLV,
+	                         MutationRef(MutationRef::ClearRange,
+	                                     startTenant.withPrefix(persistTenantMapKeys.begin),
+	                                     endTenant.withPrefix(persistTenantMapKeys.begin)));
+}
+
 ACTOR Future<Void> tssDelayForever() {
 	loop {
 		wait(delay(5.0));
@@ -5680,6 +5887,11 @@ void StorageServerDisk::makeNewStorageServerDurable() {
 	storage->set(KeyValueRef(persistVersion, BinaryWriter::toValue(data->version.get(), Unversioned())));
 	storage->set(KeyValueRef(persistShardAssignedKeys.begin.toString(), LiteralStringRef("0")));
 	storage->set(KeyValueRef(persistShardAvailableKeys.begin.toString(), LiteralStringRef("0")));
+
+	auto view = data->tenantMap.atLatest();
+	for (auto itr = view.begin(); itr != view.end(); ++itr) {
+		storage->set(KeyValueRef(itr.key().withPrefix(persistTenantMapKeys.begin), encodeTenantEntry(*itr)));
+	}
 }
 
 void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available) {
@@ -5972,6 +6184,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	state Future<RangeResult> fShardAssigned = storage->readRange(persistShardAssignedKeys);
 	state Future<RangeResult> fShardAvailable = storage->readRange(persistShardAvailableKeys);
 	state Future<RangeResult> fChangeFeeds = storage->readRange(persistChangeFeedKeys);
+	state Future<RangeResult> fTenantMap = storage->readRange(persistTenantMapKeys);
 
 	state Promise<Void> byteSampleSampleRecovered;
 	state Promise<Void> startByteSampleRestore;
@@ -5981,7 +6194,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	TraceEvent("ReadingDurableState", data->thisServerID).log();
 	wait(waitForAll(
 	    std::vector{ fFormat, fID, fClusterID, ftssPairID, fTssQuarantine, fVersion, fLogProtocol, fPrimaryLocality }));
-	wait(waitForAll(std::vector{ fShardAssigned, fShardAvailable, fChangeFeeds }));
+	wait(waitForAll(std::vector{ fShardAssigned, fShardAvailable, fChangeFeeds, fTenantMap }));
 	wait(byteSampleSampleRecovered.getFuture());
 	TraceEvent("RestoringDurableState", data->thisServerID).log();
 
@@ -6107,6 +6320,25 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		wait(yield());
 	}
 	data->keyChangeFeed.coalesce(allKeys);
+
+	state RangeResult tenantMap = fTenantMap.get();
+	state int tenantMapLoc;
+	for (tenantMapLoc = 0; tenantMapLoc < tenantMap.size(); tenantMapLoc++) {
+		auto const& result = tenantMap[tenantMapLoc];
+		TenantName tenantName = result.key.substr(persistTenantMapKeys.begin.size());
+		TenantMapEntry tenantEntry = decodeTenantEntry(result.value);
+
+		data->tenantMap.insert(tenantName, tenantEntry);
+		data->tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
+
+		TraceEvent("RestoringTenant", data->thisServerID)
+		    .detail("Key", tenantMap[tenantMapLoc].key)
+		    .detail("TenantName", tenantName)
+		    .detail("Prefix", tenantEntry.prefix);
+
+		wait(yield());
+	}
+
 	// TODO: why is this seemingly random delay here?
 	wait(delay(0.0001));
 
@@ -6424,7 +6656,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 					TEST(true); // getSplitPoints immediate wrong_shard_server()
 					self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 				} else {
-					self->metrics.getSplitPoints(req);
+					self->getSplitPoints(req);
 				}
 			}
 			when(wait(doPollMetrics)) {
@@ -6536,9 +6768,13 @@ ACTOR Future<Void> watchValueWaitForVersion(StorageServer* self,
                                             WatchValueRequest req,
                                             PromiseStream<WatchValueRequest> stream) {
 	state Span span("SS:watchValueWaitForVersion"_loc, { req.spanContext });
+	if (req.tenantInfo.name.present()) {
+		span.addTag("tenant"_sr, req.tenantInfo.name.get());
+	}
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 	try {
 		wait(success(waitForVersionNoTooOld(self, req.version)));
+		self->checkTenant(req.version, req.tenantInfo.name, req.key);
 		stream.send(req);
 	} catch (Error& e) {
 		if (!canReplyWith(e))
@@ -6589,7 +6825,8 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 			loop {
 				try {
 					state Version latest = self->version.get();
-					GetValueRequest getReq(span.context, metadata->key, latest, metadata->tags, metadata->debugID);
+					GetValueRequest getReq(
+					    span.context, TenantInfo(), metadata->key, latest, metadata->tags, metadata->debugID);
 					state Future<Void> getValue = getValueQ(self, getReq);
 					GetValueReply reply = wait(getReq.reply.getFuture());
 					metadata = self->getWatchMetadata(req.key.contents());
@@ -6886,6 +7123,27 @@ ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterC
 	}
 }
 
+ACTOR Future<Void> initTenantMap(StorageServer* self) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			state Version version = wait(tr->getReadVersion());
+			RangeResult entries = wait(tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY));
+
+			for (auto kv : entries) {
+				self->insertTenant(kv.key.removePrefix(tenantMapPrefix), kv.value, version, false);
+			}
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
+	return Void();
+}
+
 // for creating a new storage server
 ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  StorageServerInterface ssi,
@@ -6913,8 +7171,11 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 		++self.counters.kvCommits;
 
 		if (seedTag == invalidTag) {
-			std::pair<Version, Tag> verAndTag = wait(addStorageServer(
-			    self.cx, ssi)); // Might throw recruitment_failed in case of simultaneous master failure
+			wait(initTenantMap(&self));
+
+			// Might throw recruitment_failed in case of simultaneous master failure
+			std::pair<Version, Tag> verAndTag = wait(addStorageServer(self.cx, ssi));
+
 			self.tag = verAndTag.second;
 			if (ssi.isTss()) {
 				self.setInitialVersion(tssSeedVersion);
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index c25a386fdb..61d315a096 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -198,6 +198,10 @@ ERROR( client_lib_not_found, 2120, "Client library for the given identifier not
 ERROR( client_lib_not_available, 2121, "Client library exists, but is not available for download." )
 ERROR( client_lib_invalid_binary, 2122, "Invalid client library binary." )
 
+ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data in the cluster" )
+ERROR( tenant_not_found, 2131, "Tenant not found on the cluster" )
+ERROR( key_not_in_tenant, 2132, "The specified key is not in the key-range represented by the tenant" )
+
 // 2200 - errors from bindings and official APIs
 ERROR( api_version_unset, 2200, "API version is not set" )
 ERROR( api_version_already_set, 2201, "API version may be set only once" )

From 9c4773164e7ff74ed03d45499965c28d9b01411c Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Sat, 19 Feb 2022 13:32:13 -0800
Subject: [PATCH 159/210] Add ID to requests

---
 fdbclient/Tenant.h                | 17 ++++++++++++++---
 fdbserver/storageserver.actor.cpp |  4 ++--
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
index bf6cc0f1bc..ff7ee7e3fb 100644
--- a/fdbclient/Tenant.h
+++ b/fdbclient/Tenant.h
@@ -32,14 +32,25 @@ typedef Standalone<TenantNameRef> TenantName;
 struct TenantMapEntry {
 	constexpr static FileIdentifier file_identifier = 12247338;
 
+private:
+	static Key idToPrefix(int64_t id) {
+		int64_t swapped = bigEndian64(id);
+		return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
+	}
+
+public:
+	int64_t id;
 	Key prefix;
 
-	TenantMapEntry() {}
-	TenantMapEntry(Key prefix) : prefix(prefix) {}
+	TenantMapEntry() : id(-1) {}
+	TenantMapEntry(int64_t id) : id(id), prefix(idToPrefix(id)) { ASSERT(id >= 0); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, prefix);
+		serializer(ar, id);
+		if (ar.isDeserializing) {
+			prefix = idToPrefix(id);
+		}
 	}
 };
 
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index d26bbb45b1..85b8dbdfe1 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2926,7 +2926,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry("Prefix"_sr));
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry(1));
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -2934,7 +2934,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                            .append("key-2"_sr)
 		                            .append("value-0"_sr)
 		                            .getDataAsStandalone()
-		                            .withPrefix("Prefix"_sr);
+		                            .withPrefix("\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == true);

From ea273291c7d1b296c55c51ff0c0d0c454fdb24f8 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Sat, 19 Feb 2022 15:10:32 -0800
Subject: [PATCH 160/210] Add new tenant mode configuration field to status
 documentation

---
 documentation/sphinx/source/mr-status-json-schemas.rst.inc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 80302ed4f7..2e6ad4749e 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -742,6 +742,11 @@
              "disabled",
              "gradual",
              "aggressive"
+         ]},
+         "tenant_mode": {
+             "$enum":[
+             "optional",
+             "required"
          ]}
       },
       "data":{

From b8192828d9976ae4084da4d13ddeae9fb249b5cc Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Sat, 19 Feb 2022 21:10:07 -0800
Subject: [PATCH 161/210] Add support for the global tenant prefix to tenant
 map entries

---
 fdbclient/Tenant.h                | 30 ++++++++++++++++++++++++++----
 fdbserver/storageserver.actor.cpp |  4 ++--
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
index ff7ee7e3fb..19e56e9e7d 100644
--- a/fdbclient/Tenant.h
+++ b/fdbclient/Tenant.h
@@ -32,24 +32,46 @@ typedef Standalone<TenantNameRef> TenantName;
 struct TenantMapEntry {
 	constexpr static FileIdentifier file_identifier = 12247338;
 
-private:
 	static Key idToPrefix(int64_t id) {
 		int64_t swapped = bigEndian64(id);
 		return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
 	}
 
-public:
+	static int64_t prefixToId(KeyRef prefix) {
+		ASSERT(prefix.size() == 8);
+		int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
+		id = bigEndian64(id);
+		ASSERT(id >= 0);
+		return id;
+	}
+
 	int64_t id;
 	Key prefix;
 
+private:
+	void initPrefix(KeyRef subspace) {
+		ASSERT(id >= 0);
+		prefix = makeString(8 + subspace.size());
+		uint8_t* data = mutateString(prefix);
+		memcpy(data, subspace.begin(), subspace.size());
+		int64_t swapped = bigEndian64(id);
+		memcpy(data + subspace.size(), &swapped, 8);
+	}
+
+public:
 	TenantMapEntry() : id(-1) {}
-	TenantMapEntry(int64_t id) : id(id), prefix(idToPrefix(id)) { ASSERT(id >= 0); }
+	TenantMapEntry(int64_t id, KeyRef subspace) : id(id) { initPrefix(subspace); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, id);
 		if (ar.isDeserializing) {
-			prefix = idToPrefix(id);
+			KeyRef subspace;
+			serializer(ar, subspace);
+			initPrefix(subspace);
+		} else {
+			KeyRef subspace = prefix.substr(0, prefix.size() - 8);
+			serializer(ar, subspace);
 		}
 	}
 };
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 85b8dbdfe1..622552f3f0 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2926,7 +2926,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry(1));
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry(1, "foo"_sr));
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -2934,7 +2934,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                            .append("key-2"_sr)
 		                            .append("value-0"_sr)
 		                            .getDataAsStandalone()
-		                            .withPrefix("\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
+		                            .withPrefix("foo\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == true);

From bd3920c23d3f1ac6945a4e2ff8263c1b7b0b135f Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Wed, 23 Feb 2022 08:10:25 -0800
Subject: [PATCH 162/210] Wait to initialize the SS tenant map until after the
 storage server has been added. Add some extra logging around inserting and
 erasing tenants.

---
 fdbserver/storageserver.actor.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 622552f3f0..a3442c764e 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -5274,6 +5274,8 @@ void StorageServer::insertTenant(TenantNameRef tenantName,
 		addMutationToMutationLog(
 		    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
 	}
+
+	TraceEvent("InsertTenant", thisServerID).detail("Tenant", tenantName).detail("Version", version);
 }
 
 void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version) {
@@ -5285,6 +5287,7 @@ void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTen
 		// Trigger any watches on the prefix associated with the tenant.
 		watches.triggerRange(itr->prefix, strinc(itr->prefix));
 		tenantPrefixIndex.erase(itr->prefix);
+		TraceEvent("EraseTenant", thisServerID).detail("Tenant", itr.key()).detail("Version", version);
 	}
 
 	tenantMap.erase(startTenant, endTenant);
@@ -7132,6 +7135,10 @@ ACTOR Future<Void> initTenantMap(StorageServer* self) {
 			state Version version = wait(tr->getReadVersion());
 			RangeResult entries = wait(tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY));
 
+			TraceEvent("InitTenantMap", self->thisServerID)
+			    .detail("Version", version)
+			    .detail("NumTenants", entries.size());
+
 			for (auto kv : entries) {
 				self->insertTenant(kv.key.removePrefix(tenantMapPrefix), kv.value, version, false);
 			}
@@ -7171,8 +7178,6 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 		++self.counters.kvCommits;
 
 		if (seedTag == invalidTag) {
-			wait(initTenantMap(&self));
-
 			// Might throw recruitment_failed in case of simultaneous master failure
 			std::pair<Version, Tag> verAndTag = wait(addStorageServer(self.cx, ssi));
 
@@ -7182,6 +7187,8 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 			} else {
 				self.setInitialVersion(verAndTag.first - 1);
 			}
+
+			wait(initTenantMap(&self));
 		} else {
 			self.tag = seedTag;
 		}

From c99c7cf55fad3fe5d1a5ccfc144ed750a922007a Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 25 Feb 2022 11:18:42 -0800
Subject: [PATCH 163/210] Apply tenant prefixes on the storage server. The
 commit proxy can serve tenant location requests. Fix some bugs.

---
 fdbclient/CommitProxyInterface.h              |  10 +-
 fdbclient/NativeAPI.actor.cpp                 |  15 +-
 fdbclient/StorageServerInterface.cpp          |   4 +
 fdbclient/StorageServerInterface.h            |   4 +-
 fdbclient/Tenant.h                            |  15 +-
 fdbclient/VersionedMap.h                      |  12 +-
 fdbserver/ApplyMetadataMutation.cpp           |   8 +-
 fdbserver/CommitProxyServer.actor.cpp         |  92 +++--
 fdbserver/ProxyCommitData.actor.h             |   6 +-
 fdbserver/StorageMetrics.actor.h              |  32 +-
 fdbserver/storageserver.actor.cpp             | 371 +++++++++++-------
 .../workloads/ConsistencyCheck.actor.cpp      |   3 +-
 flow/error_definitions.h                      |   5 +-
 13 files changed, 362 insertions(+), 215 deletions(-)

diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index b0f1dc0e08..c50f6ba347 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -295,6 +295,7 @@ struct GetReadVersionRequest : TimedRequest {
 struct GetKeyServerLocationsReply {
 	constexpr static FileIdentifier file_identifier = 10636023;
 	Arena arena;
+	TenantMapEntry tenantEntry;
 	std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>> results;
 
 	// if any storage servers in results have a TSS pair, that mapping is in here
@@ -302,7 +303,7 @@ struct GetKeyServerLocationsReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, results, resultsTssMapping, arena);
+		serializer(ar, results, resultsTssMapping, tenantEntry, arena);
 	}
 };
 
@@ -310,6 +311,7 @@ struct GetKeyServerLocationsRequest {
 	constexpr static FileIdentifier file_identifier = 9144680;
 	Arena arena;
 	SpanID spanContext;
+	Optional<TenantNameRef> tenant;
 	KeyRef begin;
 	Optional<KeyRef> end;
 	int limit;
@@ -318,16 +320,18 @@ struct GetKeyServerLocationsRequest {
 
 	GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
 	GetKeyServerLocationsRequest(SpanID spanContext,
+	                             Optional<TenantNameRef> const& tenant,
 	                             KeyRef const& begin,
 	                             Optional<KeyRef> const& end,
 	                             int limit,
 	                             bool reverse,
 	                             Arena const& arena)
-	  : arena(arena), spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse) {}
+	  : arena(arena), spanContext(spanContext), tenant(tenant), begin(begin), end(end), limit(limit), reverse(reverse) {
+	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, begin, end, limit, reverse, reply, spanContext, arena);
+		serializer(ar, begin, end, limit, reverse, reply, spanContext, tenant, arena);
 	}
 };
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 82be820771..1f53b9290d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2545,11 +2545,13 @@ ACTOR Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation_intern
 		++cx->transactionKeyServerLocationRequests;
 		choose {
 			when(wait(cx->onProxiesChanged())) {}
-			when(GetKeyServerLocationsReply rep = wait(basicLoadBalance(
-			         cx->getCommitProxies(useProvisionalProxies),
-			         &CommitProxyInterface::getKeyServersLocations,
-			         GetKeyServerLocationsRequest(span.context, key, Optional<KeyRef>(), 100, isBackward, key.arena()),
-			         TaskPriority::DefaultPromiseEndpoint))) {
+			when(
+			    GetKeyServerLocationsReply rep = wait(basicLoadBalance(
+			        cx->getCommitProxies(useProvisionalProxies),
+			        &CommitProxyInterface::getKeyServersLocations,
+			        GetKeyServerLocationsRequest(
+			            span.context, Optional<TenantNameRef>(), key, Optional<KeyRef>(), 100, isBackward, key.arena()),
+			        TaskPriority::DefaultPromiseEndpoint))) {
 				++cx->transactionKeyServerLocationRequestsCompleted;
 				if (debugID.present())
 					g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
@@ -2650,7 +2652,8 @@ ACTOR Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRa
 			when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance(
 			         cx->getCommitProxies(useProvisionalProxies),
 			         &CommitProxyInterface::getKeyServersLocations,
-			         GetKeyServerLocationsRequest(span.context, keys.begin, keys.end, limit, reverse, keys.arena()),
+			         GetKeyServerLocationsRequest(
+			             span.context, Optional<TenantNameRef>(), keys.begin, keys.end, limit, reverse, keys.arena()),
 			         TaskPriority::DefaultPromiseEndpoint))) {
 				++cx->transactionKeyServerLocationRequestsCompleted;
 				state GetKeyServerLocationsReply rep = _rep;
diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index 11c9f46f56..3c389b5ab4 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -106,6 +106,7 @@ void TSS_traceMismatch(TraceEvent& event, const GetKeyRequest& req, const GetKey
 	event
 	    .detail("KeySelector",
 	            format("%s%s:%d", req.sel.orEqual ? "=" : "", req.sel.getKey().printable().c_str(), req.sel.offset))
+	    .detail("Tenant", req.tenantInfo.name)
 	    .detail("Version", req.version)
 	    .detail("SSReply",
 	            format("%s%s:%d", src.sel.orEqual ? "=" : "", src.sel.getKey().printable().c_str(), src.sel.offset))
@@ -144,6 +145,7 @@ void TSS_traceMismatch(TraceEvent& event,
 	        format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset))
 	    .detail("End",
 	            format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset))
+	    .detail("Tenant", req.tenantInfo.name)
 	    .detail("Version", req.version)
 	    .detail("Limit", req.limit)
 	    .detail("LimitBytes", req.limitBytes)
@@ -183,6 +185,7 @@ void TSS_traceMismatch(TraceEvent& event,
 	        format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset))
 	    .detail("End",
 	            format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset))
+	    .detail("Tenant", req.tenantInfo.name)
 	    .detail("Version", req.version)
 	    .detail("Limit", req.limit)
 	    .detail("LimitBytes", req.limitBytes)
@@ -223,6 +226,7 @@ void TSS_traceMismatch(TraceEvent& event,
 	        format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset))
 	    .detail("End",
 	            format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset))
+	    .detail("Tenant", req.tenantInfo.name)
 	    .detail("Version", req.version)
 	    .detail("Limit", req.limit)
 	    .detail("LimitBytes", req.limitBytes)
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 17927c5d9d..1aa9f120da 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -737,6 +737,7 @@ struct SplitRangeReply {
 		serializer(ar, splitPoints);
 	}
 };
+
 struct SplitRangeRequest {
 	constexpr static FileIdentifier file_identifier = 10725174;
 	Arena arena;
@@ -782,7 +783,6 @@ struct ChangeFeedStreamRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
 	SpanID spanContext;
 	Arena arena;
-	TenantInfo tenantInfo;
 	Key rangeID;
 	Version begin = 0;
 	Version end = 0;
@@ -792,7 +792,7 @@ struct ChangeFeedStreamRequest {
 	ChangeFeedStreamRequest() {}
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeID, begin, end, range, reply, spanContext, tenantInfo, arena);
+		serializer(ar, rangeID, begin, end, range, reply, spanContext, arena);
 	}
 };
 
diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
index 19e56e9e7d..1d14ea3b12 100644
--- a/fdbclient/Tenant.h
+++ b/fdbclient/Tenant.h
@@ -64,14 +64,19 @@ public:
 
 	template <class Ar>
 	void serialize(Ar& ar) {
+		KeyRef subspace;
 		serializer(ar, id);
 		if (ar.isDeserializing) {
-			KeyRef subspace;
-			serializer(ar, subspace);
-			initPrefix(subspace);
+			serializer(ar, id, subspace);
+			if (id >= 0) {
+				initPrefix(subspace);
+			}
 		} else {
-			KeyRef subspace = prefix.substr(0, prefix.size() - 8);
-			serializer(ar, subspace);
+			ASSERT(prefix.size() >= 8 || (prefix.empty() && id == -1));
+			if (!prefix.empty()) {
+				subspace = prefix.substr(0, prefix.size() - 8);
+			}
+			serializer(ar, id, subspace);
 		}
 	}
 };
diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index 1cd78a1828..2b6f3ae178 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -877,8 +877,16 @@ public:
 		Version at;
 	};
 
-	ViewAtVersion at(Version v) const { return ViewAtVersion(getRoot(v), v); }
-	ViewAtVersion atLatest() const { return ViewAtVersion(roots.back().second, latestVersion); }
+	ViewAtVersion at(Version v) const {
+		if (v == ::latestVersion) {
+			return atLatest();
+		}
+
+		return ViewAtVersion(getRoot(v), v);
+	}
+	ViewAtVersion atLatest() const {
+		return ViewAtVersion(roots.back().second, latestVersion);
+	}
 
 	bool isClearContaining(ViewAtVersion const& view, KeyRef key) {
 		auto i = view.lastLessOrEqual(key);
diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp
index e90080dca3..e0c421a22f 100644
--- a/fdbserver/ApplyMetadataMutation.cpp
+++ b/fdbserver/ApplyMetadataMutation.cpp
@@ -585,6 +585,7 @@ private:
 				Standalone<StringRef> tenantName = m.param1.removePrefix(tenantMapPrefix);
 				TenantMapEntry tenantEntry = decodeTenantEntry(m.param2);
 
+				TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version);
 				tenantMap->insert(tenantName, tenantEntry);
 			}
 
@@ -941,8 +942,13 @@ private:
 				tenantMap->createNewVersion(version);
 
 				StringRef startTenant = std::max(range.begin, tenantMapPrefix).removePrefix(tenantMapPrefix);
-				StringRef endTenant = range.end.startsWith(tenantMapPrefix) ? range.end : tenantMapKeys.end;
+				StringRef endTenant = (range.end.startsWith(tenantMapPrefix) ? range.end : tenantMapKeys.end)
+				                          .removePrefix(tenantMapPrefix);
 
+				TraceEvent("CommitProxyEraseTenants", dbgid)
+				    .detail("BeginTenant", startTenant)
+				    .detail("EndTenant", endTenant)
+				    .detail("Version", version);
 				tenantMap->erase(startTenant, endTenant);
 			}
 
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index f474b00f85..f242ea160a 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -410,6 +410,28 @@ ACTOR static Future<ResolveTransactionBatchReply> trackResolutionMetrics(Referen
 	return reply;
 }
 
+ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
+                                                 Optional<TenantNameRef> tenant,
+                                                 Version commitVersion) {
+	if (tenant.present()) {
+		auto view = commitData->tenantMap.at(commitVersion);
+		auto itr = view.find(tenant.get());
+		if (itr == view.end()) {
+			if (commitVersion != latestVersion) {
+				TraceEvent(SevWarn, "CommitProxyTenantNotFound", commitData->dbgid)
+				    .detail("Tenant", tenant.get())
+				    .detail("Version", commitVersion);
+			}
+
+			return tenant_not_found();
+		}
+
+		return ErrorOr<Optional<TenantMapEntry>>(Optional<TenantMapEntry>(*itr));
+	}
+
+	return Optional<TenantMapEntry>();
+}
+
 namespace CommitBatch {
 
 struct CommitBatchContext {
@@ -558,8 +580,8 @@ bool canReject(const std::vector<CommitTransactionRequest>& trs) {
 	for (const auto& tr : trs) {
 		if (tr.transaction.mutations.empty())
 			continue;
-		if (tr.transaction.mutations[0].param1.startsWith(LiteralStringRef("\xff")) ||
-		    tr.transaction.read_conflict_ranges.empty()) {
+		if (!tr.tenantInfo.name.present() && (tr.transaction.mutations[0].param1.startsWith(LiteralStringRef("\xff")) ||
+		                                      tr.transaction.read_conflict_ranges.empty())) {
 			return false;
 		}
 	}
@@ -670,7 +692,7 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 	state Span span("MP:getResolution"_loc, self->span.context);
 
 	ResolutionRequestBuilder requests(
-	    pProxyCommitData, self->commitVersion, self->prevVersion, pProxyCommitData->version, span);
+	    pProxyCommitData, self->commitVersion, self->prevVersion, pProxyCommitData->version.get(), span);
 	int conflictRangeCount = 0;
 	self->maxTransactionBytes = 0;
 	for (int t = 0; t < trs.size(); t++) {
@@ -837,33 +859,6 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 	}
 }
 
-ErrorOr<Void> checkTenant(ProxyCommitData* commitData, CommitTransactionRequest const& tr, Version commitVersion) {
-	if (tr.tenantInfo.name.present()) {
-		auto view = commitData->tenantMap.at(commitVersion);
-		auto itr = view.find(tr.tenantInfo.name.get());
-		if (itr == view.end()) {
-			TraceEvent(SevWarn, "CommitProxyTenantNotFound", commitData->dbgid)
-			    .detail("Tenant", tr.tenantInfo.name.get());
-			return tenant_not_found();
-		}
-
-		for (auto mutation : tr.transaction.mutations) {
-			if (!mutation.param1.startsWith(itr->prefix) ||
-			    (mutation.type == MutationRef::ClearRange && !mutation.param2.startsWith(itr->prefix))) {
-				TraceEvent(SevWarn, "KeyNotInTenant", commitData->dbgid)
-				    .detail("Tenant", tr.tenantInfo.name.get())
-				    .detail("TenantPrefix", itr->prefix)
-				    .detail("Key", mutation.param1)
-				    .detail("EndKey",
-				            mutation.type == MutationRef::ClearRange ? mutation.param2 : Optional<StringRef>());
-				throw key_not_in_tenant();
-			}
-		}
-	}
-
-	return Void();
-}
-
 // This first pass through committed transactions deals with "metadata" effects (modifications of txnStateStore, changes
 // to storage servers' responsibilities)
 ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self) {
@@ -873,7 +868,8 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	int t;
 	for (t = 0; t < trs.size() && !self->forceRecovery; t++) {
 		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) {
-			ErrorOr<Void> result = checkTenant(pProxyCommitData, trs[t], self->commitVersion);
+			ErrorOr<Optional<TenantMapEntry>> result =
+			    getTenantEntry(pProxyCommitData, trs[t].tenantInfo.name.castTo<TenantNameRef>(), self->commitVersion);
 
 			if (result.isError()) {
 				self->committed[t] = ConflictBatch::TransactionTenantFailure;
@@ -911,7 +907,7 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 
 	auto fcm = pProxyCommitData->logAdapter->getCommitMessage();
 	self->storeCommits.emplace_back(fcm, pProxyCommitData->txnStateStore->commit());
-	pProxyCommitData->version = self->commitVersion;
+	pProxyCommitData->version.set(self->commitVersion);
 	if (!pProxyCommitData->validState.isSet())
 		pProxyCommitData->validState.send(Void());
 	ASSERT(self->commitVersion);
@@ -1126,7 +1122,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 		    "CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.ProcessingMutations");
 	}
 
-	self->isMyFirstBatch = !pProxyCommitData->version;
+	self->isMyFirstBatch = !pProxyCommitData->version.get();
 	self->oldCoordinators = pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get();
 
 	assertResolutionStateMutationsSizeConsistent(self->resolution);
@@ -1530,8 +1526,36 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	wait(commitData->validState.getFuture());
 	wait(delay(0, TaskPriority::DefaultEndpoint));
 
+	state Version requestVersion = latestVersion;
+	state ErrorOr<Optional<TenantMapEntry>> tenantEntry;
+	while (tenantEntry.isError()) {
+		ErrorOr<Optional<TenantMapEntry>> _tenantEntry = getTenantEntry(commitData, req.tenant, requestVersion);
+		tenantEntry = _tenantEntry;
+
+		if (tenantEntry.isError()) {
+			if (requestVersion != latestVersion) {
+				req.reply.sendError(tenant_not_found());
+				return Void();
+			} else {
+				// If the tenant doesn't exist, wait until we are sure we've received updates from the other proxies to
+				// declare that it wasn't recently created.
+				wait(commitData->version.whenAtLeast(commitData->stats.lastCommitVersionAssigned + 1));
+				requestVersion = commitData->version.get();
+			}
+		}
+	}
+
 	std::unordered_set<UID> tssMappingsIncluded;
 	GetKeyServerLocationsReply rep;
+
+	if (tenantEntry.get().present()) {
+		rep.tenantEntry = tenantEntry.get().get();
+		req.begin = req.begin.withPrefix(rep.tenantEntry.prefix, req.arena);
+		if (req.end.present()) {
+			req.end = req.end.get().withPrefix(rep.tenantEntry.prefix, req.arena);
+		}
+	}
+
 	if (!req.end.present()) {
 		auto r = req.reverse ? commitData->keyInfo.rangeContainingKeyBefore(req.begin)
 		                     : commitData->keyInfo.rangeContaining(req.begin);
@@ -1608,7 +1632,7 @@ ACTOR static Future<Void> rejoinServer(CommitProxyInterface proxy, ProxyCommitDa
 		GetStorageServerRejoinInfoRequest req = waitNext(proxy.getStorageServerRejoinInfo.getFuture());
 		if (commitData->txnStateStore->readValue(serverListKeyFor(req.id)).get().present()) {
 			GetStorageServerRejoinInfoReply rep;
-			rep.version = commitData->version;
+			rep.version = commitData->version.get();
 			rep.tag = decodeServerTagValue(commitData->txnStateStore->readValue(serverTagKeyFor(req.id)).get().get());
 			RangeResult history = commitData->txnStateStore->readRange(serverTagHistoryRangeFor(req.id)).get();
 			for (int i = history.size() - 1; i >= 0; i--) {
diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h
index 249a7c7a60..a3518357a6 100644
--- a/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/ProxyCommitData.actor.h
@@ -97,7 +97,7 @@ struct ProxyStats {
 	}
 
 	explicit ProxyStats(UID id,
-	                    Version* pVersion,
+	                    NotifiedVersion* pVersion,
 	                    NotifiedVersion* pCommittedVersion,
 	                    int64_t* commitBatchesMemBytesCountPtr)
 	  : cc("ProxyStats", id.toString()), txnCommitIn("TxnCommitIn", cc),
@@ -146,7 +146,7 @@ struct ProxyStats {
 	                                            LiteralStringRef("ReplyCommit"),
 	                                            Histogram::Unit::microseconds)) {
 		specialCounter(cc, "LastAssignedCommitVersion", [this]() { return this->lastCommitVersionAssigned; });
-		specialCounter(cc, "Version", [pVersion]() { return *pVersion; });
+		specialCounter(cc, "Version", [pVersion]() { return pVersion->get(); });
 		specialCounter(cc, "CommittedVersion", [pCommittedVersion]() { return pCommittedVersion->get(); });
 		specialCounter(cc, "CommitBatchesMemBytesCount", [commitBatchesMemBytesCountPtr]() {
 			return *commitBatchesMemBytesCountPtr;
@@ -170,7 +170,7 @@ struct ProxyCommitData {
 	                                  // fully committed (durable)
 	Version minKnownCommittedVersion; // No version smaller than this one will be used as the known committed version
 	                                  // during recovery
-	Version version; // The version at which txnStateStore is up to date
+	NotifiedVersion version; // The version at which txnStateStore is up to date
 	Promise<Void> validState; // Set once txnStateStore and version are valid
 	double lastVersionTime;
 	KeyRangeMap<std::set<Key>> vecBackupKeys;
diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h
index 01016a3e8b..c50b5539d5 100644
--- a/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/StorageMetrics.actor.h
@@ -555,7 +555,7 @@ struct StorageServerMetrics {
 		req.reply.send(reply);
 	}
 
-	std::vector<KeyRef> getSplitPoints(KeyRangeRef range, int64_t chunkSize) {
+	std::vector<KeyRef> getSplitPoints(KeyRangeRef range, int64_t chunkSize, Optional<Key> prefixToRemove) {
 		std::vector<KeyRef> toReturn;
 		KeyRef beginKey = range.begin;
 		IndexedSet<Key, int64_t>::iterator endKey =
@@ -568,7 +568,11 @@ struct StorageServerMetrics {
 				++endKey;
 				continue;
 			}
-			toReturn.push_back(*endKey);
+			KeyRef splitPoint = *endKey;
+			if (prefixToRemove.present()) {
+				splitPoint = splitPoint.removePrefix(prefixToRemove.get());
+			}
+			toReturn.push_back(splitPoint);
 			beginKey = *endKey;
 			endKey =
 			    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize);
@@ -576,9 +580,13 @@ struct StorageServerMetrics {
 		return toReturn;
 	}
 
-	void getSplitPoints(SplitRangeRequest req) {
+	void getSplitPoints(SplitRangeRequest req, Optional<Key> prefix) {
 		SplitRangeReply reply;
-		std::vector<KeyRef> points = getSplitPoints(req.keys, req.chunkSize);
+		KeyRangeRef range = req.keys;
+		if (prefix.present()) {
+			range = range.withPrefix(prefix.get(), req.arena);
+		}
+		std::vector<KeyRef> points = getSplitPoints(range, req.chunkSize, prefix);
 
 		reply.splitPoints.append_deep(reply.splitPoints.arena(), points.data(), points.size());
 		req.reply.send(reply);
@@ -621,8 +629,8 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/simple") {
 	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
 	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
 
-	std::vector<KeyRef> t =
-	    ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 2000 * sampleUnit);
+	std::vector<KeyRef> t = ssm.getSplitPoints(
+	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 2000 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 1 && t[0] == LiteralStringRef("Bah"));
 
@@ -643,8 +651,8 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/multipleReturnedPoint
 	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
 	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
 
-	std::vector<KeyRef> t =
-	    ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 600 * sampleUnit);
+	std::vector<KeyRef> t = ssm.getSplitPoints(
+	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 600 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 3 && t[0] == LiteralStringRef("Absolute") && t[1] == LiteralStringRef("Apple") &&
 	       t[2] == LiteralStringRef("Bah"));
@@ -666,8 +674,8 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/noneSplitable") {
 	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
 	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
 
-	std::vector<KeyRef> t =
-	    ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 10000 * sampleUnit);
+	std::vector<KeyRef> t = ssm.getSplitPoints(
+	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 10000 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 0);
 
@@ -688,8 +696,8 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/chunkTooLarge") {
 	ssm.byteSample.sample.insert(LiteralStringRef("But"), 10 * sampleUnit);
 	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 30 * sampleUnit);
 
-	std::vector<KeyRef> t =
-	    ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 1000 * sampleUnit);
+	std::vector<KeyRef> t = ssm.getSplitPoints(
+	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 1000 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 0);
 
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index a3442c764e..9ca5013028 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -88,7 +88,6 @@ bool canReplyWith(Error e) {
 	case error_code_server_overloaded:
 	case error_code_tenant_name_required:
 	case error_code_tenant_not_found:
-	case error_code_key_not_in_tenant:
 	// getRangeAndMap related exceptions that are not retriable:
 	case error_code_mapper_bad_index:
 	case error_code_mapper_no_such_key:
@@ -458,10 +457,7 @@ public:
 	void insertTenant(TenantNameRef tenantName, ValueRef value, Version version, bool insertIntoMutationLog);
 	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
 
-	Optional<TenantMapEntry> checkTenant(Version version,
-	                                     Optional<TenantName> tenant,
-	                                     KeyRef begin,
-	                                     Optional<KeyRef> end);
+	Optional<TenantMapEntry> getTenantEntry(Version version, Optional<TenantName> tenant);
 	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantName> tenant, Version version, Arena& arena);
 
 	class CurrentRunningFetchKeys {
@@ -1142,8 +1138,8 @@ public:
 
 	void getSplitPoints(SplitRangeRequest const& req) {
 		try {
-			checkTenant(version.get(), req.tenantInfo.name, req.keys.begin, req.keys.end);
-			metrics.getSplitPoints(req);
+			Optional<TenantMapEntry> entry = getTenantEntry(version.get(), req.tenantInfo.name);
+			metrics.getSplitPoints(req, entry.map<Key>([](TenantMapEntry e) { return e.prefix; }));
 		} catch (Error& e) {
 			req.reply.sendError(e);
 		}
@@ -1376,32 +1372,15 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageServer* data, Version versio
 	}
 }
 
-Optional<TenantMapEntry> StorageServer::checkTenant(Version version,
-                                                    Optional<TenantName> tenantName,
-                                                    KeyRef begin,
-                                                    Optional<KeyRef> end = Optional<KeyRef>()) {
+Optional<TenantMapEntry> StorageServer::getTenantEntry(Version version, Optional<TenantName> tenantName) {
 	if (tenantName.present()) {
 		auto view = tenantMap.at(version);
 		auto itr = view.find(tenantName.get());
 		if (itr == view.end()) {
-			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID)
-			    .detail("Tenant", tenantName)
-			    .detail("Begin", begin)
-			    .detail("End", end)
-			    .backtrace();
+			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID).detail("Tenant", tenantName).backtrace();
 			throw tenant_not_found();
 		}
 
-		if (!begin.startsWith(itr->prefix) || (end.present() && !end.get().startsWith(itr->prefix))) {
-			TraceEvent(SevWarn, "KeyNotInTenant", thisServerID)
-			    .detail("Tenant", tenantName)
-			    .detail("TenantPrefix", itr->prefix)
-			    .detail("Begin", begin)
-			    .detail("End", end)
-			    .backtrace();
-			throw key_not_in_tenant();
-		}
-
 		return *itr;
 	}
 
@@ -1441,7 +1420,10 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			                      req.debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
-		data->checkTenant(version, req.tenantInfo.name, req.key);
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		if (entry.present()) {
+			req.key = req.key.withPrefix(entry.get().prefix);
+		}
 		state uint64_t changeCounter = data->shardChangeCounter;
 
 		if (!data->shards[req.key]->isReadable()) {
@@ -1858,13 +1840,12 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	if (data->version.get() < req.begin) {
 		wait(data->version.whenAtLeast(req.begin));
 	}
+
 	state uint64_t changeCounter = data->shardChangeCounter;
 	if (!inverted && !data->isReadable(req.range)) {
 		throw wrong_shard_server();
 	}
 
-	data->checkTenant(req.begin, req.tenantInfo.name, req.range.begin, req.range.end);
-
 	auto feed = data->uidChangeFeed.find(req.rangeID);
 	if (feed == data->uidChangeFeed.end()) {
 		throw unknown_change_feed();
@@ -2146,6 +2127,22 @@ ACTOR Future<Void> getShardStateQ(StorageServer* data, GetShardStateRequest req)
 	return Void();
 }
 
+KeyRef addPrefix(KeyRef const& key, Optional<Key> prefix, Arena arena) {
+	if (prefix.present()) {
+		return key.withPrefix(prefix.get(), arena);
+	} else {
+		return key;
+	}
+}
+
+KeyValueRef removePrefix(KeyValueRef const& src, Optional<Key> prefix) {
+	if (prefix.present()) {
+		return KeyValueRef(src.key.removePrefix(prefix.get()), src.value);
+	} else {
+		return src;
+	}
+}
+
 void merge(Arena& arena,
            VectorRef<KeyValueRef, VecSerStrategy::String>& output,
            VectorRef<KeyValueRef> const& vm_output,
@@ -2154,7 +2151,8 @@ void merge(Arena& arena,
            int limit,
            bool stopAtEndOfBase,
            int& pos,
-           int limitBytes = 1 << 30)
+           int limitBytes,
+           Optional<Key> tenantPrefix)
 // Combines data from base (at an older version) with sets from newer versions in [start, end) and appends the first (up
 // to) |limit| rows to output If limit<0, base and output are in descending order, and start->key()>end->key(), but
 // start is still inclusive and end is exclusive
@@ -2173,9 +2171,9 @@ void merge(Arena& arena,
 	KeyValueRef const* baseEnd = base.end();
 	while (baseStart != baseEnd && vCount > 0 && output.size() < adjustedLimit && accumulatedBytes < limitBytes) {
 		if (forward ? baseStart->key < vm_output[pos].key : baseStart->key > vm_output[pos].key) {
-			output.push_back(arena, *baseStart++);
+			output.push_back(arena, removePrefix(*baseStart++, tenantPrefix));
 		} else {
-			output.push_back_deep(arena, vm_output[pos]);
+			output.push_back_deep(arena, removePrefix(vm_output[pos], tenantPrefix));
 			if (baseStart->key == vm_output[pos].key)
 				++baseStart;
 			++pos;
@@ -2184,12 +2182,12 @@ void merge(Arena& arena,
 		accumulatedBytes += sizeof(KeyValueRef) + output.end()[-1].expectedSize();
 	}
 	while (baseStart != baseEnd && output.size() < adjustedLimit && accumulatedBytes < limitBytes) {
-		output.push_back(arena, *baseStart++);
+		output.push_back(arena, removePrefix(*baseStart++, tenantPrefix));
 		accumulatedBytes += sizeof(KeyValueRef) + output.end()[-1].expectedSize();
 	}
 	if (!stopAtEndOfBase) {
 		while (vCount > 0 && output.size() < adjustedLimit && accumulatedBytes < limitBytes) {
-			output.push_back_deep(arena, vm_output[pos]);
+			output.push_back_deep(arena, removePrefix(vm_output[pos], tenantPrefix));
 			accumulatedBytes += sizeof(KeyValueRef) + output.end()[-1].expectedSize();
 			++pos;
 			vCount--;
@@ -2246,7 +2244,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
                                           int limit,
                                           int* pLimitBytes,
                                           SpanID parentSpan,
-                                          IKeyValueStore::ReadType type) {
+                                          IKeyValueStore::ReadType type,
+                                          Optional<Key> tenantPrefix) {
 	state GetKeyValuesReply result;
 	state StorageServer::VersionedData::ViewAtVersion view = data->data().at(version);
 	state StorageServer::VersionedData::iterator vCurrent = view.end();
@@ -2301,7 +2300,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 				       vSize < *pLimitBytes) {
 					// Store the versionedData results in resultCache
 					resultCache.emplace_back(result.arena, vCurrent.key(), vCurrent->getValue());
-					vSize += sizeof(KeyValueRef) + resultCache.cback().expectedSize();
+					vSize += sizeof(KeyValueRef) + resultCache.cback().expectedSize() -
+					         (tenantPrefix.present() ? tenantPrefix.get().size() : 0);
 					++vCount;
 					++vCurrent;
 				}
@@ -2328,7 +2328,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			      limit,
 			      atStorageVersion.more,
 			      pos,
-			      *pLimitBytes);
+			      *pLimitBytes,
+			      tenantPrefix);
 			limit -= result.data.size() - prevSize;
 
 			for (auto i = result.data.begin() + prevSize; i != result.data.end(); i++) {
@@ -2341,13 +2342,22 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 			// Setup for the next iteration
 			// If we hit our limits reading from disk but then combining with MVCC gave us back more room
-			if (atStorageVersion
-			        .more) { // if there might be more data, begin reading right after what we already found to find out
-				ASSERT(result.data.end()[-1].key == atStorageVersion.end()[-1].key);
-				readBegin = readBeginTemp = keyAfter(result.data.end()[-1].key);
-			} else if (vCurrent && vCurrent->isClearTo()) { // if vCurrent is a clear, skip it.
+
+			// if there might be more data, begin reading right after what we already found to find out
+			if (atStorageVersion.more) {
+				ASSERT(atStorageVersion.end()[-1].key.size() ==
+				           result.data.end()[-1].key.size() + tenantPrefix.orDefault(""_sr).size() &&
+				       atStorageVersion.end()[-1].key.endsWith(result.data.end()[-1].key) &&
+				       atStorageVersion.end()[-1].key.startsWith(tenantPrefix.orDefault(""_sr)));
+
+				readBegin = readBeginTemp = keyAfter(atStorageVersion.end()[-1].key);
+			}
+
+			// if vCurrent is a clear, skip it.
+			else if (vCurrent && vCurrent->isClearTo()) {
 				ASSERT(vCurrent->getEndKey() > readBegin);
-				readBegin = vCurrent->getEndKey(); // next disk read should start at the end of the clear
+				// next disk read should start at the end of the clear
+				readBegin = vCurrent->getEndKey();
 				++vCurrent;
 			} else {
 				ASSERT(readEnd == range.end);
@@ -2383,7 +2393,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 				       vSize < *pLimitBytes) {
 					// Store the versionedData results in resultCache
 					resultCache.emplace_back(result.arena, vCurrent.key(), vCurrent->getValue());
-					vSize += sizeof(KeyValueRef) + resultCache.cback().expectedSize();
+					vSize += sizeof(KeyValueRef) + resultCache.cback().expectedSize() -
+					         (tenantPrefix.present() ? tenantPrefix.get().size() : 0);
 					++vCount;
 					--vCurrent;
 				}
@@ -2408,7 +2419,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			      limit,
 			      atStorageVersion.more,
 			      pos,
-			      *pLimitBytes);
+			      *pLimitBytes,
+			      tenantPrefix);
 			limit += result.data.size() - prevSize;
 
 			for (auto i = result.data.begin() + prevSize; i != result.data.end(); i++) {
@@ -2420,8 +2432,12 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			}
 
 			if (atStorageVersion.more) {
-				ASSERT(result.data.end()[-1].key == atStorageVersion.end()[-1].key);
-				readEnd = result.data.end()[-1].key;
+				ASSERT(atStorageVersion.end()[-1].key.size() ==
+				           result.data.end()[-1].key.size() + tenantPrefix.orDefault(""_sr).size() &&
+				       atStorageVersion.end()[-1].key.endsWith(result.data.end()[-1].key) &&
+				       atStorageVersion.end()[-1].key.startsWith(tenantPrefix.orDefault(""_sr)));
+
+				readEnd = atStorageVersion.end()[-1].key;
 			} else if (vCurrent && vCurrent->isClearTo()) {
 				ASSERT(vCurrent.key() < readEnd);
 				readEnd = vCurrent.key();
@@ -2501,7 +2517,8 @@ ACTOR Future<Key> findKey(StorageServer* data,
 	              (distance + skipEqualKey) * sign,
 	              &maxBytes,
 	              span.context,
-	              type));
+	              type,
+	              Optional<Key>()));
 	state bool more = rep.more && rep.data.size() != distance + skipEqualKey;
 
 	// If we get only one result in the reverse direction as a result of the data being too large, we could get stuck in
@@ -2509,8 +2526,14 @@ ACTOR Future<Key> findKey(StorageServer* data,
 	if (more && !forward && rep.data.size() == 1) {
 		TEST(true); // Reverse key selector returned only one result in range read
 		maxBytes = std::numeric_limits<int>::max();
-		GetKeyValuesReply rep2 = wait(readRange(
-		    data, version, KeyRangeRef(range.begin, keyAfter(sel.getKey())), -2, &maxBytes, span.context, type));
+		GetKeyValuesReply rep2 = wait(readRange(data,
+		                                        version,
+		                                        KeyRangeRef(range.begin, keyAfter(sel.getKey())),
+		                                        -2,
+		                                        &maxBytes,
+		                                        span.context,
+		                                        type,
+		                                        Optional<Key>()));
 		rep = rep2;
 		more = rep.more && rep.data.size() != distance + skipEqualKey;
 		ASSERT(rep.data.size() == 2 || !more);
@@ -2603,7 +2626,15 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		data->checkTenant(version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		if (tenantPrefix.present()) {
+			req.begin = KeySelectorRef(
+			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
+			req.end = KeySelectorRef(
+			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+		}
+
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
@@ -2672,8 +2703,14 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		} else {
 			state int remainingLimitBytes = req.limitBytes;
 
-			GetKeyValuesReply _r = wait(
-			    readRange(data, version, KeyRangeRef(begin, end), req.limit, &remainingLimitBytes, span.context, type));
+			GetKeyValuesReply _r = wait(readRange(data,
+			                                      version,
+			                                      KeyRangeRef(begin, end),
+			                                      req.limit,
+			                                      &remainingLimitBytes,
+			                                      span.context,
+			                                      type,
+			                                      tenantPrefix));
 			GetKeyValuesReply r = _r;
 
 			if (req.debugID.present())
@@ -2685,18 +2722,17 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			    KeyRangeRef(std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())),
 			                std::max<KeyRef>(end, std::max<KeyRef>(req.begin.getKey(), req.end.getKey()))));
 			if (EXPENSIVE_VALIDATION) {
-				for (int i = 0; i < r.data.size(); i++)
-					ASSERT(r.data[i].key >= begin && r.data[i].key < end);
+				for (int i = 0; i < r.data.size(); i++) {
+					if (tenantPrefix.present()) {
+						ASSERT(r.data[i].key >= begin.removePrefix(tenantPrefix.get()) &&
+						       r.data[i].key < end.removePrefix(tenantPrefix.get()));
+					} else {
+						ASSERT(r.data[i].key >= begin && r.data[i].key < end);
+					}
+				}
 				ASSERT(r.data.size() <= std::abs(req.limit));
 			}
 
-			/*for( int i = 0; i < r.data.size(); i++ ) {
-			    StorageMetrics m;
-			    m.bytesPerKSecond = r.data[i].expectedSize();
-			    m.iosPerKSecond = 1; //FIXME: this should be 1/r.data.size(), but we cannot do that because it is an int
-			    data->metrics.notify(r.data[i].key, m);
-			}*/
-
 			// For performance concerns, the cost of a range read is billed to the start key and end key of the range.
 			int64_t totalByteSize = 0;
 			for (int i = 0; i < r.data.size(); i++) {
@@ -2704,8 +2740,10 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			}
 			if (totalByteSize > 0 && SERVER_KNOBS->READ_SAMPLING_ENABLED) {
 				int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
-				data->metrics.notifyBytesReadPerKSecond(r.data[0].key, bytesReadPerKSecond);
-				data->metrics.notifyBytesReadPerKSecond(r.data[r.data.size() - 1].key, bytesReadPerKSecond);
+				data->metrics.notifyBytesReadPerKSecond(addPrefix(r.data[0].key, tenantPrefix, req.arena),
+				                                        bytesReadPerKSecond);
+				data->metrics.notifyBytesReadPerKSecond(
+				    addPrefix(r.data[r.data.size() - 1].key, tenantPrefix, req.arena), bytesReadPerKSecond);
 			}
 
 			r.penalty = data->getPenalty();
@@ -2793,7 +2831,7 @@ ACTOR Future<RangeResult> quickGetKeyValues(StorageServer* data,
 Key constructMappedKey(KeyValueRef* keyValue,
                        Tuple& mappedKeyFormatTuple,
                        bool& isRangeQuery,
-                       Optional<TenantMapEntry> tenantEntry) {
+                       Optional<Key> tenantPrefix) {
 	// Lazily parse key and/or value to tuple because they may not need to be a tuple if not used.
 	Optional<Tuple> keyTuple;
 	Optional<Tuple> valueTuple;
@@ -2885,8 +2923,8 @@ Key constructMappedKey(KeyValueRef* keyValue,
 
 	KeyRef mappedKey = mappedKeyTuple.pack();
 
-	if (tenantEntry.present() && !tenantEntry.get().prefix.empty()) {
-		return mappedKey.withPrefix(tenantEntry.get().prefix);
+	if (tenantPrefix.present()) {
+		return mappedKey.withPrefix(tenantPrefix.get());
 	}
 
 	return mappedKey;
@@ -2905,7 +2943,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -2926,7 +2964,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry(1, "foo"_sr));
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, "foo"_sr);
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -2943,7 +2981,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		Tuple mapperTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 
 		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
@@ -2954,7 +2992,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		Tuple mapperTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 
 		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
@@ -2966,7 +3004,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException = true;
@@ -2978,7 +3016,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException2 = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_range_decriptor);
 			throwException2 = true;
@@ -2990,7 +3028,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		bool isRangeQuery = false;
 		state bool throwException3 = false;
 		try {
-			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<TenantMapEntry>());
+			Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, Optional<Key>());
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException3 = true;
@@ -3005,7 +3043,7 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
                                                   StringRef mapper,
                                                   // To provide span context, tags, debug ID to underlying lookups.
                                                   GetKeyValuesAndFlatMapRequest* pOriginalReq,
-                                                  Optional<TenantMapEntry> tenantEntry) {
+                                                  Optional<Key> tenantPrefix) {
 	state GetKeyValuesAndFlatMapReply result;
 	result.version = input.version;
 	if (input.more) {
@@ -3020,7 +3058,7 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
 	state Tuple mappedKeyFormatTuple = Tuple::unpack(mapper);
 	state KeyValueRef* it = input.data.begin();
 	for (; it != input.data.end(); it++) {
-		state Key mappedKey = constructMappedKey(it, mappedKeyFormatTuple, isRangeQuery, tenantEntry);
+		state Key mappedKey = constructMappedKey(it, mappedKeyFormatTuple, isRangeQuery, tenantPrefix);
 		// Make sure the mappedKey is always available, so that it's good even we want to get key asynchronously.
 		result.arena.dependsOn(mappedKey.arena());
 
@@ -3034,7 +3072,11 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
 			}
 			result.arena.dependsOn(rangeResult.arena());
 			for (int i = 0; i < rangeResult.size(); i++) {
-				result.data.emplace_back(result.arena, rangeResult[i].key, rangeResult[i].value);
+				KeyRef key = rangeResult[i].key;
+				if (tenantPrefix.present()) {
+					key = key.removePrefix(tenantPrefix.get());
+				}
+				result.data.emplace_back(result.arena, key, rangeResult[i].value);
 			}
 		} else {
 			Optional<Value> valueOption = wait(quickGetValue(data, mappedKey, input.version, pOriginalReq));
@@ -3042,7 +3084,12 @@ ACTOR Future<GetKeyValuesAndFlatMapReply> flatMap(StorageServer* data,
 			if (valueOption.present()) {
 				Value value = valueOption.get();
 				result.arena.dependsOn(value.arena());
-				result.data.emplace_back(result.arena, mappedKey, value);
+
+				KeyRef key = mappedKey;
+				if (tenantPrefix.present()) {
+					key = key.removePrefix(tenantPrefix.get());
+				}
+				result.data.emplace_back(result.arena, key, value);
 			} else {
 				// TODO: Shall we throw exception if the key doesn't exist or the range is empty?
 			}
@@ -3088,8 +3135,14 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesAndFlatMap.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		state Optional<TenantMapEntry> tenantEntry =
-		    data->checkTenant(req.version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
+		Optional<TenantMapEntry> entry = data->getTenantEntry(req.version, req.tenantInfo.name);
+		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		if (tenantPrefix.present()) {
+			req.begin = KeySelectorRef(
+			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
+			req.end = KeySelectorRef(
+			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+		}
 
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
@@ -3123,7 +3176,7 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 
-		if (!tenantEntry.present()) {
+		if (!tenantPrefix.present()) {
 			// We do not support raw flat map requests when using tenants.
 			// When tenants are required, we disable raw flat map requests entirely.
 			// If tenants are optional, we check whether the range intersects a tenant and fail if it does.
@@ -3179,13 +3232,19 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 		} else {
 			state int remainingLimitBytes = req.limitBytes;
 
-			GetKeyValuesReply getKeyValuesReply = wait(
-			    readRange(data, version, KeyRangeRef(begin, end), req.limit, &remainingLimitBytes, span.context, type));
+			GetKeyValuesReply getKeyValuesReply = wait(readRange(data,
+			                                                     version,
+			                                                     KeyRangeRef(begin, end),
+			                                                     req.limit,
+			                                                     &remainingLimitBytes,
+			                                                     span.context,
+			                                                     type,
+			                                                     tenantPrefix));
 
 			state GetKeyValuesAndFlatMapReply r;
 			try {
 				// Map the scanned range to another list of keys and look up.
-				GetKeyValuesAndFlatMapReply _r = wait(flatMap(data, getKeyValuesReply, req.mapper, &req, tenantEntry));
+				GetKeyValuesAndFlatMapReply _r = wait(flatMap(data, getKeyValuesReply, req.mapper, &req, tenantPrefix));
 				r = _r;
 			} catch (Error& e) {
 				TraceEvent("FlatMapError").error(e);
@@ -3202,20 +3261,10 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			    KeyRangeRef(std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())),
 			                std::max<KeyRef>(end, std::max<KeyRef>(req.begin.getKey(), req.end.getKey()))));
 			if (EXPENSIVE_VALIDATION) {
-				// TODO: Only mapped keys are returned, which are not supposed to be in the range.
-				//				for (int i = 0; i < r.data.size(); i++)
-				//					ASSERT(r.data[i].key >= begin && r.data[i].key < end);
 				// TODO: GetKeyValuesWithFlatMapRequest doesn't respect limit yet.
 				//                ASSERT(r.data.size() <= std::abs(req.limit));
 			}
 
-			/*for( int i = 0; i < r.data.size(); i++ ) {
-			    StorageMetrics m;
-			    m.bytesPerKSecond = r.data[i].expectedSize();
-			    m.iosPerKSecond = 1; //FIXME: this should be 1/r.data.size(), but we cannot do that because it is an int
-			    data->metrics.notify(r.data[i].key, m);
-			}*/
-
 			// For performance concerns, the cost of a range read is billed to the start key and end key of the range.
 			int64_t totalByteSize = 0;
 			for (int i = 0; i < r.data.size(); i++) {
@@ -3223,8 +3272,10 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			}
 			if (totalByteSize > 0 && SERVER_KNOBS->READ_SAMPLING_ENABLED) {
 				int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
-				data->metrics.notifyBytesReadPerKSecond(r.data[0].key, bytesReadPerKSecond);
-				data->metrics.notifyBytesReadPerKSecond(r.data[r.data.size() - 1].key, bytesReadPerKSecond);
+				data->metrics.notifyBytesReadPerKSecond(addPrefix(r.data[0].key, tenantPrefix, req.arena),
+				                                        bytesReadPerKSecond);
+				data->metrics.notifyBytesReadPerKSecond(
+				    addPrefix(r.data[r.data.size() - 1].key, tenantPrefix, req.arena), bytesReadPerKSecond);
 			}
 
 			r.penalty = data->getPenalty();
@@ -3297,7 +3348,15 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		data->checkTenant(req.version, req.tenantInfo.name, req.begin.getKey(), req.end.getKey());
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		if (tenantPrefix.present()) {
+			req.begin = KeySelectorRef(
+			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
+			req.end = KeySelectorRef(
+			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+		}
+
 		state uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
@@ -3377,8 +3436,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				                       !data->isTss() && !data->isSSWithTSSPair())
 				                          ? 1
 				                          : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-				GetKeyValuesReply _r =
-				    wait(readRange(data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, type));
+				GetKeyValuesReply _r = wait(readRange(
+				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, type, tenantPrefix));
 				GetKeyValuesStreamReply r(_r);
 
 				if (req.debugID.present())
@@ -3391,28 +3450,33 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				    KeyRangeRef(std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())),
 				                std::max<KeyRef>(end, std::max<KeyRef>(req.begin.getKey(), req.end.getKey()))));
 				if (EXPENSIVE_VALIDATION) {
-					for (int i = 0; i < r.data.size(); i++)
-						ASSERT(r.data[i].key >= begin && r.data[i].key < end);
+					for (int i = 0; i < r.data.size(); i++) {
+						if (tenantPrefix.present()) {
+							ASSERT(r.data[i].key >= begin.removePrefix(tenantPrefix.get()) &&
+							       r.data[i].key < end.removePrefix(tenantPrefix.get()));
+						} else {
+							ASSERT(r.data[i].key >= begin && r.data[i].key < end);
+						}
+					}
 					ASSERT(r.data.size() <= std::abs(req.limit));
 				}
 
-				/*for( int i = 0; i < r.data.size(); i++ ) {
-				    StorageMetrics m;
-				    m.bytesPerKSecond = r.data[i].expectedSize();
-				    m.iosPerKSecond = 1; //FIXME: this should be 1/r.data.size(), but we cannot do that because it is an
-				int data->metrics.notify(r.data[i].key, m);
-				}*/
-
 				// For performance concerns, the cost of a range read is billed to the start key and end key of the
 				// range.
 				int64_t totalByteSize = 0;
 				for (int i = 0; i < r.data.size(); i++) {
 					totalByteSize += r.data[i].expectedSize();
 				}
+
+				KeyRef lastKey;
+				if (!r.data.empty()) {
+					lastKey = addPrefix(r.data.back().key, tenantPrefix, req.arena);
+				}
 				if (totalByteSize > 0 && SERVER_KNOBS->READ_SAMPLING_ENABLED) {
 					int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
-					data->metrics.notifyBytesReadPerKSecond(r.data[0].key, bytesReadPerKSecond);
-					data->metrics.notifyBytesReadPerKSecond(r.data[r.data.size() - 1].key, bytesReadPerKSecond);
+					KeyRef firstKey = addPrefix(r.data[0].key, tenantPrefix, req.arena);
+					data->metrics.notifyBytesReadPerKSecond(firstKey, bytesReadPerKSecond);
+					data->metrics.notifyBytesReadPerKSecond(lastKey, bytesReadPerKSecond);
 				}
 
 				req.reply.send(r);
@@ -3428,9 +3492,9 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				ASSERT(r.data.size());
 
 				if (req.limit >= 0) {
-					begin = keyAfter(r.data.back().key);
+					begin = keyAfter(lastKey);
 				} else {
-					end = r.data.back().key;
+					end = lastKey;
 				}
 
 				if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
@@ -3478,18 +3542,28 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	try {
 		state Version version = wait(waitForVersion(data, req.version, req.spanContext));
 
-		data->checkTenant(version, req.tenantInfo.name, req.sel.getKey());
+		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo.name);
+		if (tenantEntry.present()) {
+			req.sel = KeySelectorRef(
+			    req.sel.getKey().withPrefix(tenantEntry.get().prefix, req.arena), req.sel.orEqual, req.sel.offset);
+		}
 		state uint64_t changeCounter = data->shardChangeCounter;
 
 		KeyRange shard = getShardKeyRange(data, req.sel);
 		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, req.version, req.arena);
 
 		state int offset;
-		Key k = wait(
+		Key absoluteKey = wait(
 		    findKey(data, req.sel, version, searchRange, &offset, req.spanContext, IKeyValueStore::ReadType::NORMAL));
 
-		data->checkChangeCounter(
-		    changeCounter, KeyRangeRef(std::min<KeyRef>(req.sel.getKey(), k), std::max<KeyRef>(req.sel.getKey(), k)));
+		data->checkChangeCounter(changeCounter,
+		                         KeyRangeRef(std::min<KeyRef>(req.sel.getKey(), absoluteKey),
+		                                     std::max<KeyRef>(req.sel.getKey(), absoluteKey)));
+
+		KeyRef k = absoluteKey;
+		if (tenantEntry.present()) {
+			k = k.removePrefix(tenantEntry.get().prefix);
+		}
 
 		KeySelector updated;
 		if (offset < 0)
@@ -3507,7 +3581,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		++data->counters.rowsQueried;
 
 		// Check if the desired key might be cached
-		auto cached = data->cachedRangeMap[k];
+		auto cached = data->cachedRangeMap[absoluteKey];
 		// if (cached)
 		//	TraceEvent(SevDebug, "SSGetKeyCached").detail("Key", k).detail("Begin",
 		// shard.begin.printable()).detail("End", shard.end.printable());
@@ -5261,42 +5335,46 @@ void StorageServer::insertTenant(TenantNameRef tenantName,
                                  ValueRef value,
                                  Version version,
                                  bool insertIntoMutationLog) {
-	tenantMap.createNewVersion(version);
-	tenantPrefixIndex.createNewVersion(version);
+	if (version >= tenantMap.getLatestVersion()) {
+		tenantMap.createNewVersion(version);
+		tenantPrefixIndex.createNewVersion(version);
 
-	TenantMapEntry tenantEntry = decodeTenantEntry(value);
+		TenantMapEntry tenantEntry = decodeTenantEntry(value);
 
-	tenantMap.insert(tenantName, tenantEntry);
-	tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
+		tenantMap.insert(tenantName, tenantEntry);
+		tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
 
-	if (insertIntoMutationLog) {
-		auto& mLV = addVersionToMutationLog(version);
-		addMutationToMutationLog(
-		    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
+		if (insertIntoMutationLog) {
+			auto& mLV = addVersionToMutationLog(version);
+			addMutationToMutationLog(
+			    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
+		}
+
+		TraceEvent("InsertTenant", thisServerID).detail("Tenant", tenantName).detail("Version", version);
 	}
-
-	TraceEvent("InsertTenant", thisServerID).detail("Tenant", tenantName).detail("Version", version);
 }
 
 void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version) {
-	tenantMap.createNewVersion(version);
-	tenantPrefixIndex.createNewVersion(version);
+	if (version >= tenantMap.getLatestVersion()) {
+		tenantMap.createNewVersion(version);
+		tenantPrefixIndex.createNewVersion(version);
 
-	auto view = tenantMap.at(version);
-	for (auto itr = view.lower_bound(startTenant); itr != view.lower_bound(endTenant); ++itr) {
-		// Trigger any watches on the prefix associated with the tenant.
-		watches.triggerRange(itr->prefix, strinc(itr->prefix));
-		tenantPrefixIndex.erase(itr->prefix);
-		TraceEvent("EraseTenant", thisServerID).detail("Tenant", itr.key()).detail("Version", version);
+		auto view = tenantMap.at(version);
+		for (auto itr = view.lower_bound(startTenant); itr != view.lower_bound(endTenant); ++itr) {
+			// Trigger any watches on the prefix associated with the tenant.
+			watches.triggerRange(itr->prefix, strinc(itr->prefix));
+			tenantPrefixIndex.erase(itr->prefix);
+			TraceEvent("EraseTenant", thisServerID).detail("Tenant", itr.key()).detail("Version", version);
+		}
+
+		tenantMap.erase(startTenant, endTenant);
+
+		auto& mLV = addVersionToMutationLog(version);
+		addMutationToMutationLog(mLV,
+		                         MutationRef(MutationRef::ClearRange,
+		                                     startTenant.withPrefix(persistTenantMapKeys.begin),
+		                                     endTenant.withPrefix(persistTenantMapKeys.begin)));
 	}
-
-	tenantMap.erase(startTenant, endTenant);
-
-	auto& mLV = addVersionToMutationLog(version);
-	addMutationToMutationLog(mLV,
-	                         MutationRef(MutationRef::ClearRange,
-	                                     startTenant.withPrefix(persistTenantMapKeys.begin),
-	                                     endTenant.withPrefix(persistTenantMapKeys.begin)));
 }
 
 ACTOR Future<Void> tssDelayForever() {
@@ -6777,7 +6855,10 @@ ACTOR Future<Void> watchValueWaitForVersion(StorageServer* self,
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 	try {
 		wait(success(waitForVersionNoTooOld(self, req.version)));
-		self->checkTenant(req.version, req.tenantInfo.name, req.key);
+		Optional<TenantMapEntry> entry = self->getTenantEntry(req.version, req.tenantInfo.name);
+		if (entry.present()) {
+			req.key = req.key.withPrefix(entry.get().prefix);
+		}
 		stream.send(req);
 	} catch (Error& e) {
 		if (!canReplyWith(e))
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index d7b526b6f6..c3d55d406e 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -882,7 +882,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				keyServerLocationFutures.push_back(
 				    commitProxyInfo->get(i, &CommitProxyInterface::getKeyServersLocations)
 				        .getReplyUnlessFailedFor(
-				            GetKeyServerLocationsRequest(span.context, begin, end, limitKeyServers, false, Arena()),
+				            GetKeyServerLocationsRequest(
+				                span.context, Optional<TenantNameRef>(), begin, end, limitKeyServers, false, Arena()),
 				            2,
 				            0));
 
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index 61d315a096..85ef4f7098 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -200,7 +200,10 @@ ERROR( client_lib_invalid_binary, 2122, "Invalid client library binary." )
 
 ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data in the cluster" )
 ERROR( tenant_not_found, 2131, "Tenant not found on the cluster" )
-ERROR( key_not_in_tenant, 2132, "The specified key is not in the key-range represented by the tenant" )
+ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists" )
+ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
+ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
+ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
 
 // 2200 - errors from bindings and official APIs
 ERROR( api_version_unset, 2200, "API version is not set" )

From e898e59d18abc3d60a5690d45415586c62492bc7 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 1 Mar 2022 08:27:00 -0800
Subject: [PATCH 164/210] Fix merge issue that caused tenant serialization
 problems. Fix case where arena was passed by value rather than reference.
 Watches should check the tenant map at the latest version rather than the
 watch version.

---
 fdbclient/Tenant.h                |  1 -
 fdbserver/storageserver.actor.cpp | 26 ++++++++++++++++----------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
index 1d14ea3b12..7c5750e0dd 100644
--- a/fdbclient/Tenant.h
+++ b/fdbclient/Tenant.h
@@ -65,7 +65,6 @@ public:
 	template <class Ar>
 	void serialize(Ar& ar) {
 		KeyRef subspace;
-		serializer(ar, id);
 		if (ar.isDeserializing) {
 			serializer(ar, id, subspace);
 			if (id >= 0) {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 9ca5013028..13f744ef18 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2127,7 +2127,7 @@ ACTOR Future<Void> getShardStateQ(StorageServer* data, GetShardStateRequest req)
 	return Void();
 }
 
-KeyRef addPrefix(KeyRef const& key, Optional<Key> prefix, Arena arena) {
+KeyRef addPrefix(KeyRef const& key, Optional<Key> prefix, Arena& arena) {
 	if (prefix.present()) {
 		return key.withPrefix(prefix.get(), arena);
 	} else {
@@ -6855,7 +6855,7 @@ ACTOR Future<Void> watchValueWaitForVersion(StorageServer* self,
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 	try {
 		wait(success(waitForVersionNoTooOld(self, req.version)));
-		Optional<TenantMapEntry> entry = self->getTenantEntry(req.version, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = self->getTenantEntry(latestVersion, req.tenantInfo.name);
 		if (entry.present()) {
 			req.key = req.key.withPrefix(entry.get().prefix);
 		}
@@ -6876,21 +6876,24 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 		state Span span("SS:serveWatchValueRequestsImpl"_loc, { req.spanContext });
 		getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 
-		if (!metadata.isValid()) { // case 1: no watch set for the current key
+		// case 1: no watch set for the current key
+		if (!metadata.isValid()) {
 			metadata = makeReference<ServerWatchMetadata>(req.key, req.value, req.version, req.tags, req.debugID);
 			KeyRef key = self->setWatchMetadata(metadata);
 			metadata->watch_impl = forward(watchWaitForValueChange(self, span.context, key), metadata->versionPromise);
 			self->actors.add(watchValueSendReply(self, req, metadata->versionPromise.getFuture(), span.context));
-		} else if (metadata->value ==
-		           req.value) { // case 2: there is a watch in the map and it has the same value so just update version
+		}
+		// case 2: there is a watch in the map and it has the same value so just update version
+		else if (metadata->value == req.value) {
 			if (req.version > metadata->version) {
 				metadata->version = req.version;
 				metadata->tags = req.tags;
 				metadata->debugID = req.debugID;
 			}
 			self->actors.add(watchValueSendReply(self, req, metadata->versionPromise.getFuture(), span.context));
-		} else if (req.version > metadata->version) { // case 3: version in map has a lower version so trigger watch and
-			                                          // create a new entry in map
+		}
+		// case 3: version in map has a lower version so trigger watch and create a new entry in map
+		else if (req.version > metadata->version) {
 			self->deleteWatchMetadata(req.key.contents());
 			metadata->versionPromise.send(req.version);
 			metadata->watch_impl.cancel();
@@ -6900,11 +6903,14 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 			metadata->watch_impl = forward(watchWaitForValueChange(self, span.context, key), metadata->versionPromise);
 
 			self->actors.add(watchValueSendReply(self, req, metadata->versionPromise.getFuture(), span.context));
-		} else if (req.version <
-		           metadata->version) { // case 4: version in the map is higher so immediately trigger watch
+		}
+		// case 4: version in the map is higher so immediately trigger watch
+		else if (req.version < metadata->version) {
 			TEST(true); // watch version in map is higher so trigger watch (case 4)
 			req.reply.send(WatchValueReply{ metadata->version });
-		} else { // case 5: watch value differs but their versions are the same (rare case) so check with the SS
+		}
+		// case 5: watch value differs but their versions are the same (rare case) so check with the SS
+		else {
 			TEST(true); // watch version in the map is the same but value is different (case 5)
 			loop {
 				try {

From c630cb45cd656760166b1b5744e5444edd938228 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 1 Mar 2022 09:39:56 -0800
Subject: [PATCH 165/210] Fix formatting

---
 fdbclient/VersionedMap.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index 2b6f3ae178..5744d90743 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -884,9 +884,7 @@ public:
 
 		return ViewAtVersion(getRoot(v), v);
 	}
-	ViewAtVersion atLatest() const {
-		return ViewAtVersion(roots.back().second, latestVersion);
-	}
+	ViewAtVersion atLatest() const { return ViewAtVersion(roots.back().second, latestVersion); }
 
 	bool isClearContaining(ViewAtVersion const& view, KeyRef key) {
 		auto i = view.lastLessOrEqual(key);

From b4feddea8ddc1b913aa92321a9a2e29ff9eca4ff Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Wed, 2 Mar 2022 10:13:31 -0800
Subject: [PATCH 166/210] Fix a couple bugs. Don't store old versions in tenant
 map indefinitely.

---
 fdbclient/Tenant.h                |  2 +-
 fdbserver/storageserver.actor.cpp | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fdbclient/Tenant.h b/fdbclient/Tenant.h
index 7c5750e0dd..221ae451bb 100644
--- a/fdbclient/Tenant.h
+++ b/fdbclient/Tenant.h
@@ -81,6 +81,6 @@ public:
 };
 
 typedef VersionedMap<TenantName, TenantMapEntry> TenantMap;
-typedef VersionedMap<KeyRef, TenantNameRef> TenantPrefixIndex;
+typedef VersionedMap<Key, TenantName> TenantPrefixIndex;
 
 #endif
\ No newline at end of file
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 13f744ef18..0489e9fb47 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2964,7 +2964,7 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		                        .append("{...}"_sr);
 
 		bool isRangeQuery = false;
-		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, "foo"_sr);
+		Key mappedKey = constructMappedKey(&kvr, mapperTuple, isRangeQuery, TenantMapEntry(1, "foo"_sr).prefix);
 
 		Key expectedMappedKey = Tuple()
 		                            .append("normal"_sr)
@@ -5831,11 +5831,17 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 		state double beforeStorageUpdates = now();
 		loop {
 			state bool done = data->storage.makeVersionMutationsDurable(newOldestVersion, desiredVersion, bytesLeft);
+			if (data->tenantMap.getLatestVersion() < newOldestVersion) {
+				data->tenantMap.createNewVersion(newOldestVersion);
+				data->tenantPrefixIndex.createNewVersion(newOldestVersion);
+			}
 			// We want to forget things from these data structures atomically with changing oldestVersion (and "before",
 			// since oldestVersion.set() may trigger waiting actors) forgetVersionsBeforeAsync visibly forgets
 			// immediately (without waiting) but asynchronously frees memory.
 			Future<Void> finishedForgetting =
-			    data->mutableData().forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
+			    data->mutableData().forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
+			    data->tenantMap.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
+			    data->tenantPrefixIndex.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
 			data->oldestVersion.set(newOldestVersion);
 			wait(finishedForgetting);
 			wait(yield(TaskPriority::UpdateStorage));

From 72a34945ce986e2466479ef5d10934adb4e85ada Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Thu, 3 Mar 2022 15:40:38 -0800
Subject: [PATCH 167/210] Add the ability to disable tenants. Server processes
 verify the ID of tenants being read or written.

---
 .../source/mr-status-json-schemas.rst.inc     |  5 +--
 fdbclient/DatabaseConfiguration.cpp           |  4 +--
 fdbclient/FDBTypes.h                          | 12 ++++---
 fdbclient/ManagementAPI.actor.cpp             |  8 +++--
 fdbclient/Schemas.cpp                         |  5 +--
 fdbclient/StorageServerInterface.h            | 12 ++++---
 fdbserver/CommitProxyServer.actor.cpp         | 20 ++++++++++--
 fdbserver/storageserver.actor.cpp             | 31 ++++++++++++-------
 flow/error_definitions.h                      |  1 +
 9 files changed, 64 insertions(+), 34 deletions(-)

diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 2e6ad4749e..c9e42402a1 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -745,8 +745,9 @@
          ]},
          "tenant_mode": {
              "$enum":[
-             "optional",
-             "required"
+             "disabled",
+             "optional_experimental",
+             "required_experimental"
          ]}
       },
       "data":{
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index d84682953f..5c44db08d8 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -50,7 +50,7 @@ void DatabaseConfiguration::resetInternal() {
 	perpetualStorageWiggleSpeed = 0;
 	perpetualStorageWiggleLocality = "0";
 	storageMigrationType = StorageMigrationType::DEFAULT;
-	tenantMode = TenantMode::OPTIONAL;
+	tenantMode = TenantMode::DISABLED;
 }
 
 int toInt(ValueRef const& v) {
@@ -210,7 +210,7 @@ bool DatabaseConfiguration::isValid() const {
 	      // We cannot specify regions with three_datacenter replication
 	      (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) &&
 	      isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality) &&
-	      storageMigrationType != StorageMigrationType::UNSET && tenantMode >= TenantMode::OPTIONAL &&
+	      storageMigrationType != StorageMigrationType::UNSET && tenantMode >= TenantMode::DISABLED &&
 	      tenantMode < TenantMode::END)) {
 		return false;
 	}
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index c51afa6530..001a048e30 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1187,12 +1187,12 @@ struct StorageMigrationType {
 struct TenantMode {
 	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones
 	// just before END.
-	enum Mode { OPTIONAL = 0, REQUIRED = 1, END = 2 };
+	enum Mode { DISABLED = 0, OPTIONAL = 1, REQUIRED = 2, END = 3 };
 
-	TenantMode() : mode(OPTIONAL) {}
+	TenantMode() : mode(DISABLED) {}
 	TenantMode(Mode mode) : mode(mode) {
 		if ((uint32_t)mode >= END) {
-			this->mode = OPTIONAL;
+			this->mode = DISABLED;
 		}
 	}
 	operator Mode() const { return Mode(mode); }
@@ -1204,10 +1204,12 @@ struct TenantMode {
 
 	std::string toString() const {
 		switch (mode) {
+		case DISABLED:
+			return "disabled";
 		case OPTIONAL:
-			return "optional";
+			return "optional_experimental";
 		case REQUIRED:
-			return "required";
+			return "required_experimental";
 		default:
 			ASSERT(false);
 		}
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 6f8f9350c7..e88d2044bb 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -176,12 +176,14 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 		}
 		if (key == "tenant_mode") {
 			TenantMode tenantMode;
-			if (value == "optional") {
+			if (value == "disabled") {
+				tenantMode = TenantMode::DISABLED;
+			} else if (value == "optional_experimental") {
 				tenantMode = TenantMode::OPTIONAL;
-			} else if (value == "required") {
+			} else if (value == "required_experimental") {
 				tenantMode = TenantMode::REQUIRED;
 			} else {
-				printf("Error: Only optional|required are valid for tenant_mode.\n");
+				printf("Error: Only disabled|optional_experimental|required_experimental are valid for tenant_mode.\n");
 				return out;
 			}
 			out[p + key] = format("%d", tenantMode);
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 3988202e93..0a40b7832c 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -812,8 +812,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
          ]},
          "tenant_mode": {
              "$enum":[
-             "optional",
-             "required"
+             "disabled",
+             "optional_experimental",
+             "required_experimental"
          ]}
       },
       "data":{
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 1aa9f120da..aef7486c3d 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -214,15 +214,17 @@ struct ServerCacheInfo {
 };
 
 struct TenantInfo {
-	Optional<TenantName> name;
+	static const int64_t INVALID_TENANT = -1;
 
-	TenantInfo() {}
-	TenantInfo(Optional<TenantName> name) : name(name) {}
-	TenantInfo(TenantName name) : name(name) {}
+	Optional<TenantName> name;
+	int64_t tenantId;
+
+	TenantInfo() : tenantId(INVALID_TENANT) {}
+	TenantInfo(TenantName name, int64_t tenantId) : name(name), tenantId(tenantId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, name);
+		serializer(ar, name, tenantId);
 	}
 };
 
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index f242ea160a..8673f4597b 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -412,6 +412,7 @@ ACTOR static Future<ResolveTransactionBatchReply> trackResolutionMetrics(Referen
 
 ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
                                                  Optional<TenantNameRef> tenant,
+                                                 Optional<int64_t> tenantId,
                                                  Version commitVersion) {
 	if (tenant.present()) {
 		auto view = commitData->tenantMap.at(commitVersion);
@@ -423,6 +424,16 @@ ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
 				    .detail("Version", commitVersion);
 			}
 
+			return tenant_not_found();
+		} else if (tenantId.present() && tenantId.get() != itr->id) {
+			if (commitVersion != latestVersion) {
+				TraceEvent(SevWarn, "CommitProxyTenantIdMismatch", commitData->dbgid)
+				    .detail("Tenant", tenant.get())
+				    .detail("TenantId", tenantId)
+				    .detail("ExistingId", itr->id)
+				    .detail("Version", commitVersion);
+			}
+
 			return tenant_not_found();
 		}
 
@@ -868,8 +879,10 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	int t;
 	for (t = 0; t < trs.size() && !self->forceRecovery; t++) {
 		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) {
-			ErrorOr<Optional<TenantMapEntry>> result =
-			    getTenantEntry(pProxyCommitData, trs[t].tenantInfo.name.castTo<TenantNameRef>(), self->commitVersion);
+			ErrorOr<Optional<TenantMapEntry>> result = getTenantEntry(pProxyCommitData,
+			                                                          trs[t].tenantInfo.name.castTo<TenantNameRef>(),
+			                                                          trs[t].tenantInfo.tenantId,
+			                                                          self->commitVersion);
 
 			if (result.isError()) {
 				self->committed[t] = ConflictBatch::TransactionTenantFailure;
@@ -1529,7 +1542,8 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	state Version requestVersion = latestVersion;
 	state ErrorOr<Optional<TenantMapEntry>> tenantEntry;
 	while (tenantEntry.isError()) {
-		ErrorOr<Optional<TenantMapEntry>> _tenantEntry = getTenantEntry(commitData, req.tenant, requestVersion);
+		ErrorOr<Optional<TenantMapEntry>> _tenantEntry =
+		    getTenantEntry(commitData, req.tenant, Optional<int64_t>(), requestVersion);
 		tenantEntry = _tenantEntry;
 
 		if (tenantEntry.isError()) {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 0489e9fb47..28288163c7 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -457,7 +457,7 @@ public:
 	void insertTenant(TenantNameRef tenantName, ValueRef value, Version version, bool insertIntoMutationLog);
 	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
 
-	Optional<TenantMapEntry> getTenantEntry(Version version, Optional<TenantName> tenant);
+	Optional<TenantMapEntry> getTenantEntry(Version version, TenantInfo tenant);
 	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantName> tenant, Version version, Arena& arena);
 
 	class CurrentRunningFetchKeys {
@@ -1138,7 +1138,7 @@ public:
 
 	void getSplitPoints(SplitRangeRequest const& req) {
 		try {
-			Optional<TenantMapEntry> entry = getTenantEntry(version.get(), req.tenantInfo.name);
+			Optional<TenantMapEntry> entry = getTenantEntry(version.get(), req.tenantInfo);
 			metrics.getSplitPoints(req, entry.map<Key>([](TenantMapEntry e) { return e.prefix; }));
 		} catch (Error& e) {
 			req.reply.sendError(e);
@@ -1372,12 +1372,19 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageServer* data, Version versio
 	}
 }
 
-Optional<TenantMapEntry> StorageServer::getTenantEntry(Version version, Optional<TenantName> tenantName) {
-	if (tenantName.present()) {
+Optional<TenantMapEntry> StorageServer::getTenantEntry(Version version, TenantInfo tenantInfo) {
+	if (tenantInfo.name.present()) {
 		auto view = tenantMap.at(version);
-		auto itr = view.find(tenantName.get());
+		auto itr = view.find(tenantInfo.name.get());
 		if (itr == view.end()) {
-			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID).detail("Tenant", tenantName).backtrace();
+			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID).detail("Tenant", tenantInfo.name).backtrace();
+			throw tenant_not_found();
+		} else if (itr->id != tenantInfo.tenantId) {
+			TraceEvent(SevWarn, "StorageTenantIdMismatch", thisServerID)
+			    .detail("Tenant", tenantInfo.name)
+			    .detail("TenantId", tenantInfo.tenantId)
+			    .detail("ExistingId", itr->id)
+			    .backtrace();
 			throw tenant_not_found();
 		}
 
@@ -1420,7 +1427,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			                      req.debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
 		if (entry.present()) {
 			req.key = req.key.withPrefix(entry.get().prefix);
 		}
@@ -2626,7 +2633,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin = KeySelectorRef(
@@ -3135,7 +3142,7 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesAndFlatMap.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(req.version, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = data->getTenantEntry(req.version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin = KeySelectorRef(
@@ -3348,7 +3355,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin = KeySelectorRef(
@@ -3542,7 +3549,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	try {
 		state Version version = wait(waitForVersion(data, req.version, req.spanContext));
 
-		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo.name);
+		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
 		if (tenantEntry.present()) {
 			req.sel = KeySelectorRef(
 			    req.sel.getKey().withPrefix(tenantEntry.get().prefix, req.arena), req.sel.orEqual, req.sel.offset);
@@ -6861,7 +6868,7 @@ ACTOR Future<Void> watchValueWaitForVersion(StorageServer* self,
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
 	try {
 		wait(success(waitForVersionNoTooOld(self, req.version)));
-		Optional<TenantMapEntry> entry = self->getTenantEntry(latestVersion, req.tenantInfo.name);
+		Optional<TenantMapEntry> entry = self->getTenantEntry(latestVersion, req.tenantInfo);
 		if (entry.present()) {
 			req.key = req.key.withPrefix(entry.get().prefix);
 		}
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index 85ef4f7098..ba86a2aa17 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -204,6 +204,7 @@ ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists
 ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
 ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
 ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
+ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster");
 
 // 2200 - errors from bindings and official APIs
 ERROR( api_version_unset, 2200, "API version is not set" )

From 3ae98189af93478d82d91eebef5b2e6e9ff94d31 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Fri, 4 Mar 2022 12:22:05 -0800
Subject: [PATCH 168/210] Server processes send unknown_tenant responses back
 to clients, which is meant to be retried after updating the tenant cache. Fix
 bug where key selectors could be truncated after applying the tenant prefix.

---
 bindings/c/fdb_c.cpp                          |  3 +-
 .../sphinx/source/api-error-codes.rst         | 14 +++++++++
 fdbclient/FDBTypes.cpp                        |  4 +++
 fdbclient/FDBTypes.h                          |  1 +
 fdbserver/CommitProxyServer.actor.cpp         |  6 ++--
 fdbserver/storageserver.actor.cpp             | 29 +++++++------------
 flow/error_definitions.h                      |  3 +-
 7 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index 612f454125..baa4caf521 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -76,7 +76,8 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro
 		return code == error_code_not_committed || code == error_code_transaction_too_old ||
 		       code == error_code_future_version || code == error_code_database_locked ||
 		       code == error_code_proxy_memory_limit_exceeded || code == error_code_batch_transaction_throttled ||
-		       code == error_code_process_behind || code == error_code_tag_throttled;
+		       code == error_code_process_behind || code == error_code_tag_throttled ||
+		       code == error_code_unknown_tenant;
 	}
 	return false;
 }
diff --git a/documentation/sphinx/source/api-error-codes.rst b/documentation/sphinx/source/api-error-codes.rst
index e8f564d80d..b9c72df92e 100644
--- a/documentation/sphinx/source/api-error-codes.rst
+++ b/documentation/sphinx/source/api-error-codes.rst
@@ -160,6 +160,20 @@ FoundationDB may return the following error codes from API functions. If you nee
 | special_keys_api_failure                      | 2117| Api call through special keys failed. For more information, read the           |
 |                                               |     | ``0xff0xff/error_message`` key                                                 |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenant_name_required                          | 2130| Tenant name must be specified to access data in the cluster                    |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenant_not_found                              | 2131| Tenant does not exist                                                          |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenant_already_exists                         | 2132| A tenant with the given name already exists                                    |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenant_not_empty                              | 2133| Cannot delete a non-empty tenant                                               |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| invalid_tenant_name                           | 2134| Tenant name cannot begin with \xff                                             |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenant_prefix_allocator_conflict              | 2135| The database already has keys stored at the prefix allocated for the tenant    |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| tenants_disabled                              | 2136| Tenants have been disabled in the cluster                                      |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | api_version_unset                             | 2200| API version is not set                                                         |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | api_version_already_set                       | 2201| API version may be set only once                                               |
diff --git a/fdbclient/FDBTypes.cpp b/fdbclient/FDBTypes.cpp
index 8ada7f2c08..17390365ed 100644
--- a/fdbclient/FDBTypes.cpp
+++ b/fdbclient/FDBTypes.cpp
@@ -52,6 +52,10 @@ void KeySelectorRef::setKey(KeyRef const& key) {
 		this->key = key;
 }
 
+void KeySelectorRef::setKeyUnlimited(KeyRef const& key) {
+	this->key = key;
+}
+
 std::string KeySelectorRef::toString() const {
 	if (offset > 0) {
 		if (orEqual)
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 001a048e30..ea53601792 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -550,6 +550,7 @@ public:
 	KeyRef getKey() const { return key; }
 
 	void setKey(KeyRef const& key);
+	void setKeyUnlimited(KeyRef const& key);
 
 	std::string toString() const;
 
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 8673f4597b..0ffbbc3a24 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -419,12 +419,12 @@ ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
 		auto itr = view.find(tenant.get());
 		if (itr == view.end()) {
 			if (commitVersion != latestVersion) {
-				TraceEvent(SevWarn, "CommitProxyTenantNotFound", commitData->dbgid)
+				TraceEvent(SevWarn, "CommitProxyUnknownTenant", commitData->dbgid)
 				    .detail("Tenant", tenant.get())
 				    .detail("Version", commitVersion);
 			}
 
-			return tenant_not_found();
+			return unknown_tenant();
 		} else if (tenantId.present() && tenantId.get() != itr->id) {
 			if (commitVersion != latestVersion) {
 				TraceEvent(SevWarn, "CommitProxyTenantIdMismatch", commitData->dbgid)
@@ -434,7 +434,7 @@ ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
 				    .detail("Version", commitVersion);
 			}
 
-			return tenant_not_found();
+			return unknown_tenant();
 		}
 
 		return ErrorOr<Optional<TenantMapEntry>>(Optional<TenantMapEntry>(*itr));
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 28288163c7..0839449cca 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -87,7 +87,7 @@ bool canReplyWith(Error e) {
 	case error_code_unknown_change_feed:
 	case error_code_server_overloaded:
 	case error_code_tenant_name_required:
-	case error_code_tenant_not_found:
+	case error_code_unknown_tenant:
 	// getRangeAndMap related exceptions that are not retriable:
 	case error_code_mapper_bad_index:
 	case error_code_mapper_no_such_key:
@@ -1377,15 +1377,15 @@ Optional<TenantMapEntry> StorageServer::getTenantEntry(Version version, TenantIn
 		auto view = tenantMap.at(version);
 		auto itr = view.find(tenantInfo.name.get());
 		if (itr == view.end()) {
-			TraceEvent(SevWarn, "StorageTenantNotFound", thisServerID).detail("Tenant", tenantInfo.name).backtrace();
-			throw tenant_not_found();
+			TraceEvent(SevWarn, "StorageUnknownTenant", thisServerID).detail("Tenant", tenantInfo.name).backtrace();
+			throw unknown_tenant();
 		} else if (itr->id != tenantInfo.tenantId) {
 			TraceEvent(SevWarn, "StorageTenantIdMismatch", thisServerID)
 			    .detail("Tenant", tenantInfo.name)
 			    .detail("TenantId", tenantInfo.tenantId)
 			    .detail("ExistingId", itr->id)
 			    .backtrace();
-			throw tenant_not_found();
+			throw unknown_tenant();
 		}
 
 		return *itr;
@@ -2636,10 +2636,8 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
-			req.begin = KeySelectorRef(
-			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
-			req.end = KeySelectorRef(
-			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
+			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
 		}
 
 		state uint64_t changeCounter = data->shardChangeCounter;
@@ -3145,10 +3143,8 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 		Optional<TenantMapEntry> entry = data->getTenantEntry(req.version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
-			req.begin = KeySelectorRef(
-			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
-			req.end = KeySelectorRef(
-			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
+			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
 		}
 
 		state uint64_t changeCounter = data->shardChangeCounter;
@@ -3358,10 +3354,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
-			req.begin = KeySelectorRef(
-			    req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena), req.begin.orEqual, req.begin.offset);
-			req.end = KeySelectorRef(
-			    req.end.getKey().withPrefix(tenantPrefix.get(), req.arena), req.end.orEqual, req.end.offset);
+			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
+			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
 		}
 
 		state uint64_t changeCounter = data->shardChangeCounter;
@@ -3551,8 +3545,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 
 		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
 		if (tenantEntry.present()) {
-			req.sel = KeySelectorRef(
-			    req.sel.getKey().withPrefix(tenantEntry.get().prefix, req.arena), req.sel.orEqual, req.sel.offset);
+			req.sel.setKeyUnlimited(req.sel.getKey().withPrefix(tenantEntry.get().prefix, req.arena));
 		}
 		state uint64_t changeCounter = data->shardChangeCounter;
 
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index ba86a2aa17..15a3df8fde 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -199,12 +199,13 @@ ERROR( client_lib_not_available, 2121, "Client library exists, but is not availa
 ERROR( client_lib_invalid_binary, 2122, "Invalid client library binary." )
 
 ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data in the cluster" )
-ERROR( tenant_not_found, 2131, "Tenant not found on the cluster" )
+ERROR( tenant_not_found, 2131, "Tenant does not exist" )
 ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists" )
 ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
 ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
 ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
 ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster");
+ERROR( unknown_tenant, 2137, "Tenant is not available from this server")
 
 // 2200 - errors from bindings and official APIs
 ERROR( api_version_unset, 2200, "API version is not set" )

From 665e182bcc19dad7a733602bb286aa7955a55b23 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 16 Feb 2022 08:07:17 -0600
Subject: [PATCH 169/210] TSS Metrics improvements

---
 fdbclient/ClientKnobs.cpp     |   1 +
 fdbclient/ClientKnobs.h       |   1 +
 fdbclient/NativeAPI.actor.cpp | 135 +++++++++++++++++++---------------
 fdbrpc/ContinuousSample.h     |   4 +-
 fdbrpc/TSSComparison.h        |   4 +-
 5 files changed, 85 insertions(+), 60 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 974912c0d6..7e8ae46cc4 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -36,6 +36,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( SYSTEM_MONITOR_INTERVAL,                 5.0 );
 	init( NETWORK_BUSYNESS_MONITOR_INTERVAL,       1.0 );
+	init( TSS_METRICS_LOGGING_INTERVAL,          120.0 ); // 2 minutes by default
 
 	init( FAILURE_MAX_DELAY,                       5.0 );
 	init( FAILURE_MIN_DELAY,                       4.0 ); if( randomize && BUGGIFY ) FAILURE_MIN_DELAY = 1.0;
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 212bca21be..7b01e1c1c0 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -35,6 +35,7 @@ public:
 
 	double SYSTEM_MONITOR_INTERVAL;
 	double NETWORK_BUSYNESS_MONITOR_INTERVAL; // The interval in which we should update the network busyness metric
+	double TSS_METRICS_LOGGING_INTERVAL;
 
 	double FAILURE_MAX_DELAY;
 	double FAILURE_MIN_DELAY;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9edd325680..1a134b386a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -409,6 +409,82 @@ void traceTSSErrors(const char* name, UID tssId, const std::unordered_map<int, u
 	}
 }
 
+/*
+    For each request type, this will produce
+    <Type>Count
+    <Type>{SS,TSS}{Mean,P50,P90,P99}
+    Example:
+    GetValueLatencySSMean
+*/
+void traceSSOrTSSPercentiles(TraceEvent& ev, const std::string name, ContinuousSample<double>& sample) {
+	ev.detail(name + "Mean", sample.mean());
+	ev.detail(name + "P50", sample.median());
+	// don't log the larger percentiles unless we actually have enough samples to log the accurate percentile instead of
+	// the largest sample in this window
+	if (sample.getPopulationSize() > 10) {
+		ev.detail(name + "P90", sample.percentile(0.90));
+	}
+	if (sample.getPopulationSize() > 100) {
+		ev.detail(name + "P99", sample.percentile(0.99));
+	}
+}
+
+void traceTSSPercentiles(TraceEvent& ev,
+                         const std::string name,
+                         ContinuousSample<double>& ssSample,
+                         ContinuousSample<double>& tssSample) {
+	ASSERT(ssSample.getPopulationSize() == tssSample.getPopulationSize());
+	ev.detail(name + "Count", ssSample.getPopulationSize());
+	if (ssSample.getPopulationSize() > 0) {
+		traceSSOrTSSPercentiles(ev, name + "SS", ssSample);
+		traceSSOrTSSPercentiles(ev, name + "TSS", tssSample);
+	}
+}
+
+ACTOR Future<Void> tssLogger(DatabaseContext* cx) {
+	state double lastLogged = 0;
+	loop {
+		wait(delay(CLIENT_KNOBS->TSS_METRICS_LOGGING_INTERVAL, TaskPriority::FlushTrace));
+
+		// Log each TSS pair separately
+		for (const auto& it : cx->tssMetrics) {
+			if (it.second->mismatches.getIntervalDelta()) {
+				cx->tssMismatchStream.send(
+				    std::pair<UID, std::vector<DetailedTSSMismatch>>(it.first, it.second->detailedMismatches));
+			}
+
+			// Do error histograms as separate event
+			if (it.second->ssErrorsByCode.size()) {
+				traceTSSErrors("TSS_SSErrors", it.first, it.second->ssErrorsByCode);
+			}
+
+			if (it.second->tssErrorsByCode.size()) {
+				traceTSSErrors("TSS_TSSErrors", it.first, it.second->tssErrorsByCode);
+			}
+
+			TraceEvent tssEv("TSSClientMetrics", cx->dbId);
+			tssEv.detail("TSSID", it.first)
+			    .detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)
+			    .detail("Internal", cx->internal);
+
+			it.second->cc.logToTraceEvent(tssEv);
+
+			traceTSSPercentiles(tssEv, "GetValueLatency", it.second->SSgetValueLatency, it.second->TSSgetValueLatency);
+			traceTSSPercentiles(
+			    tssEv, "GetKeyValuesLatency", it.second->SSgetKeyValuesLatency, it.second->TSSgetKeyValuesLatency);
+			traceTSSPercentiles(tssEv, "GetKeyLatency", it.second->SSgetKeyLatency, it.second->TSSgetKeyLatency);
+			traceTSSPercentiles(tssEv,
+			                    "GetKeyValuesAndFlatMapLatency",
+			                    it.second->SSgetKeyValuesAndFlatMapLatency,
+			                    it.second->TSSgetKeyValuesAndFlatMapLatency);
+
+			it.second->clear();
+		}
+
+		lastLogged = now();
+	}
+}
+
 ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 	state double lastLogged = 0;
 	loop {
@@ -455,63 +531,6 @@ ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 		cx->mutationsPerCommit.clear();
 		cx->bytesPerCommit.clear();
 
-		for (const auto& it : cx->tssMetrics) {
-			// TODO could skip this whole thing if tss if request counter is zero?
-			// That would potentially complicate elapsed calculation though
-			if (it.second->mismatches.getIntervalDelta()) {
-				cx->tssMismatchStream.send(
-				    std::pair<UID, std::vector<DetailedTSSMismatch>>(it.first, it.second->detailedMismatches));
-			}
-
-			// do error histograms as separate event
-			if (it.second->ssErrorsByCode.size()) {
-				traceTSSErrors("TSS_SSErrors", it.first, it.second->ssErrorsByCode);
-			}
-
-			if (it.second->tssErrorsByCode.size()) {
-				traceTSSErrors("TSS_TSSErrors", it.first, it.second->tssErrorsByCode);
-			}
-
-			TraceEvent tssEv("TSSClientMetrics", cx->dbId);
-			tssEv.detail("TSSID", it.first)
-			    .detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)
-			    .detail("Internal", cx->internal);
-
-			it.second->cc.logToTraceEvent(tssEv);
-
-			tssEv.detail("MeanSSGetValueLatency", it.second->SSgetValueLatency.mean())
-			    .detail("MedianSSGetValueLatency", it.second->SSgetValueLatency.median())
-			    .detail("SSGetValueLatency90", it.second->SSgetValueLatency.percentile(0.90))
-			    .detail("SSGetValueLatency99", it.second->SSgetValueLatency.percentile(0.99));
-
-			tssEv.detail("MeanTSSGetValueLatency", it.second->TSSgetValueLatency.mean())
-			    .detail("MedianTSSGetValueLatency", it.second->TSSgetValueLatency.median())
-			    .detail("TSSGetValueLatency90", it.second->TSSgetValueLatency.percentile(0.90))
-			    .detail("TSSGetValueLatency99", it.second->TSSgetValueLatency.percentile(0.99));
-
-			tssEv.detail("MeanSSGetKeyLatency", it.second->SSgetKeyLatency.mean())
-			    .detail("MedianSSGetKeyLatency", it.second->SSgetKeyLatency.median())
-			    .detail("SSGetKeyLatency90", it.second->SSgetKeyLatency.percentile(0.90))
-			    .detail("SSGetKeyLatency99", it.second->SSgetKeyLatency.percentile(0.99));
-
-			tssEv.detail("MeanTSSGetKeyLatency", it.second->TSSgetKeyLatency.mean())
-			    .detail("MedianTSSGetKeyLatency", it.second->TSSgetKeyLatency.median())
-			    .detail("TSSGetKeyLatency90", it.second->TSSgetKeyLatency.percentile(0.90))
-			    .detail("TSSGetKeyLatency99", it.second->TSSgetKeyLatency.percentile(0.99));
-
-			tssEv.detail("MeanSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.mean())
-			    .detail("MedianSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.median())
-			    .detail("SSGetKeyValuesLatency90", it.second->SSgetKeyValuesLatency.percentile(0.90))
-			    .detail("SSGetKeyValuesLatency99", it.second->SSgetKeyValuesLatency.percentile(0.99));
-
-			tssEv.detail("MeanTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.mean())
-			    .detail("MedianTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.median())
-			    .detail("TSSGetKeyValuesLatency90", it.second->TSSgetKeyValuesLatency.percentile(0.90))
-			    .detail("TSSGetKeyValuesLatency99", it.second->TSSgetKeyValuesLatency.percentile(0.99));
-
-			it.second->clear();
-		}
-
 		lastLogged = now();
 	}
 }
@@ -1319,7 +1338,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 
 	snapshotRywEnabled = apiVersionAtLeast(300) ? 1 : 0;
 
-	logger = databaseLogger(this);
+	logger = databaseLogger(this) && tssLogger(this);
 	locationCacheSize = g_network->isSimulated() ? CLIENT_KNOBS->LOCATION_CACHE_EVICTION_SIZE_SIM
 	                                             : CLIENT_KNOBS->LOCATION_CACHE_EVICTION_SIZE;
 
diff --git a/fdbrpc/ContinuousSample.h b/fdbrpc/ContinuousSample.h
index 064dbbe6b6..0c90ade044 100644
--- a/fdbrpc/ContinuousSample.h
+++ b/fdbrpc/ContinuousSample.h
@@ -66,7 +66,9 @@ public:
 		if (!samples.size() || percentile < 0.0 || percentile > 1.0)
 			return T();
 		sort();
-		int idx = std::floor((samples.size() - 1) * percentile);
+		// Percentile (X) is the smallest element in the sample set at least as large as X% of the samples.
+		// Min with last idx to ensure no out of bounds for percentile=1.0
+		int idx = std::min((int)std::ceil(samples.size() - 1 * percentile), (int)samples.size() - 1);
 		return samples[idx];
 	}
 
diff --git a/fdbrpc/TSSComparison.h b/fdbrpc/TSSComparison.h
index 54114c2db8..88c453a7b0 100644
--- a/fdbrpc/TSSComparison.h
+++ b/fdbrpc/TSSComparison.h
@@ -40,7 +40,7 @@ struct DetailedTSSMismatch {
 
 struct TSSMetrics : ReferenceCounted<TSSMetrics>, NonCopyable {
 	CounterCollection cc;
-	Counter requests;
+	Counter requests; // requests is the number of requests attempted, successful or not
 	Counter streamComparisons;
 	Counter ssErrors;
 	Counter tssErrors;
@@ -90,10 +90,12 @@ struct TSSMetrics : ReferenceCounted<TSSMetrics>, NonCopyable {
 		SSgetValueLatency.clear();
 		SSgetKeyLatency.clear();
 		SSgetKeyValuesLatency.clear();
+		SSgetKeyValuesAndFlatMapLatency.clear();
 
 		TSSgetValueLatency.clear();
 		TSSgetKeyLatency.clear();
 		TSSgetKeyValuesLatency.clear();
+		TSSgetKeyValuesAndFlatMapLatency.clear();
 
 		tssErrorsByCode.clear();
 		ssErrorsByCode.clear();

From 04f678d515f5aa13a16ff4bf1dbe62ec2159bb6f Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 16 Feb 2022 12:04:34 -0600
Subject: [PATCH 170/210] Fixing sample logic

---
 fdbclient/NativeAPI.actor.cpp | 8 +++++---
 fdbrpc/ContinuousSample.h     | 5 ++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 1a134b386a..b7de9217c1 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -418,13 +418,15 @@ void traceTSSErrors(const char* name, UID tssId, const std::unordered_map<int, u
 */
 void traceSSOrTSSPercentiles(TraceEvent& ev, const std::string name, ContinuousSample<double>& sample) {
 	ev.detail(name + "Mean", sample.mean());
-	ev.detail(name + "P50", sample.median());
 	// don't log the larger percentiles unless we actually have enough samples to log the accurate percentile instead of
 	// the largest sample in this window
-	if (sample.getPopulationSize() > 10) {
+	if (sample.getPopulationSize() >= 2) {
+		ev.detail(name + "P50", sample.median());
+	}
+	if (sample.getPopulationSize() >= 10) {
 		ev.detail(name + "P90", sample.percentile(0.90));
 	}
-	if (sample.getPopulationSize() > 100) {
+	if (sample.getPopulationSize() >= 100) {
 		ev.detail(name + "P99", sample.percentile(0.99));
 	}
 }
diff --git a/fdbrpc/ContinuousSample.h b/fdbrpc/ContinuousSample.h
index 0c90ade044..b8cc25aee0 100644
--- a/fdbrpc/ContinuousSample.h
+++ b/fdbrpc/ContinuousSample.h
@@ -62,13 +62,12 @@ public:
 
 	T median() { return percentile(0.5); }
 
+	// Percentile (X) is the smallest element in the sample set at least as large as X% of the samples.
 	T percentile(double percentile) {
 		if (!samples.size() || percentile < 0.0 || percentile > 1.0)
 			return T();
 		sort();
-		// Percentile (X) is the smallest element in the sample set at least as large as X% of the samples.
-		// Min with last idx to ensure no out of bounds for percentile=1.0
-		int idx = std::min((int)std::ceil(samples.size() - 1 * percentile), (int)samples.size() - 1);
+		int idx = std::max<int>(0, std::ceil(samples.size() * percentile) - 1);
 		return samples[idx];
 	}
 

From 58f5cea39bef3fa60557cd7f9ef40133604494dc Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 16 Feb 2022 13:29:38 -0600
Subject: [PATCH 171/210] fix typo

---
 fdbclient/NativeAPI.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index b7de9217c1..33e582443f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -420,7 +420,7 @@ void traceSSOrTSSPercentiles(TraceEvent& ev, const std::string name, ContinuousS
 	ev.detail(name + "Mean", sample.mean());
 	// don't log the larger percentiles unless we actually have enough samples to log the accurate percentile instead of
 	// the largest sample in this window
-	if (sample.getPopulationSize() >= 2) {
+	if (sample.getPopulationSize() >= 3) {
 		ev.detail(name + "P50", sample.median());
 	}
 	if (sample.getPopulationSize() >= 10) {

From 07169b1f116297ff107becd4ce08f9fd699a859b Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Mon, 7 Mar 2022 10:37:22 -0800
Subject: [PATCH 172/210] Add unknown_tenant error to documentation. Avoid
 doing an extra tenant lookup in the clampRangeToTenant function.

---
 .../sphinx/source/api-error-codes.rst         |  2 +
 fdbserver/storageserver.actor.cpp             | 41 ++++++++++---------
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/documentation/sphinx/source/api-error-codes.rst b/documentation/sphinx/source/api-error-codes.rst
index b9c72df92e..7c06dbb6df 100644
--- a/documentation/sphinx/source/api-error-codes.rst
+++ b/documentation/sphinx/source/api-error-codes.rst
@@ -174,6 +174,8 @@ FoundationDB may return the following error codes from API functions. If you nee
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | tenants_disabled                              | 2136| Tenants have been disabled in the cluster                                      |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| unknown_tenant                                | 2137| Tenant is not available from this server                                       |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | api_version_unset                             | 2200| API version is not set                                                         |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | api_version_already_set                       | 2201| API version may be set only once                                               |
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 0839449cca..aef837df52 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -458,7 +458,10 @@ public:
 	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
 
 	Optional<TenantMapEntry> getTenantEntry(Version version, TenantInfo tenant);
-	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantName> tenant, Version version, Arena& arena);
+	KeyRangeRef clampRangeToTenant(KeyRangeRef range,
+	                               Optional<TenantMapEntry> tenantEntry,
+	                               Version version,
+	                               Arena& arena);
 
 	class CurrentRunningFetchKeys {
 		std::unordered_map<UID, double> startTimeMap;
@@ -2464,16 +2467,14 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 }
 
 KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range,
-                                              Optional<TenantName> tenant,
+                                              Optional<TenantMapEntry> tenantEntry,
                                               Version version,
                                               Arena& arena) {
-	if (tenant.present()) {
-		auto view = tenantMap.at(version);
-		auto itr = view.find(tenant.get());
-		ASSERT(itr != view.end());
-
-		return KeyRangeRef(range.begin.startsWith(itr->prefix) ? range.begin : itr->prefix,
-		                   range.end.startsWith(itr->prefix) ? range.end : allKeys.end.withPrefix(itr->prefix, arena));
+	if (tenantEntry.present()) {
+		return KeyRangeRef(range.begin.startsWith(tenantEntry.get().prefix) ? range.begin : tenantEntry.get().prefix,
+		                   range.end.startsWith(tenantEntry.get().prefix)
+		                       ? range.end
+		                       : allKeys.end.withPrefix(tenantEntry.get().prefix, arena));
 	} else {
 		return range;
 	}
@@ -2633,8 +2634,8 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
-		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
+		state Optional<Key> tenantPrefix = tenantEntry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
 			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
@@ -2659,7 +2660,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3140,8 +3141,8 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesAndFlatMap.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(req.version, req.tenantInfo);
-		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(req.version, req.tenantInfo);
+		state Optional<Key> tenantPrefix = tenantEntry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
 			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
@@ -3166,7 +3167,7 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3351,8 +3352,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 		state Version version = wait(waitForVersion(data, req.version, span.context));
 
-		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
-		state Optional<Key> tenantPrefix = entry.map<Key>([](TenantMapEntry e) { return e.prefix; });
+		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
+		state Optional<Key> tenantPrefix = tenantEntry.map<Key>([](TenantMapEntry e) { return e.prefix; });
 		if (tenantPrefix.present()) {
 			req.begin.setKeyUnlimited(req.begin.getKey().withPrefix(tenantPrefix.get(), req.arena));
 			req.end.setKeyUnlimited(req.end.getKey().withPrefix(tenantPrefix.get(), req.arena));
@@ -3377,7 +3378,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3550,7 +3551,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		state uint64_t changeCounter = data->shardChangeCounter;
 
 		KeyRange shard = getShardKeyRange(data, req.sel);
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo.name, req.version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.version, req.arena);
 
 		state int offset;
 		Key absoluteKey = wait(
@@ -7226,6 +7227,8 @@ ACTOR Future<Void> initTenantMap(StorageServer* self) {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			state Version version = wait(tr->getReadVersion());
+			// This limits the number of tenants, but eventually we shouldn't need to do this at all
+			// when SSs store only the local tenants
 			RangeResult entries = wait(tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY));
 
 			TraceEvent("InitTenantMap", self->thisServerID)

From 8f8f95931b9905d93a37a8c5b2dd6265df4d1396 Mon Sep 17 00:00:00 2001
From: Steve Atherton <steve.atherton@snowflake.com>
Date: Mon, 7 Mar 2022 08:41:44 -0800
Subject: [PATCH 173/210] In the SQLite storage engine, destroy and create new
 cursors after SQLITE_CURSOR_MAX_LIFETIME_BYTES of KV read usage because
 cursor usage increases page cache memory usage in SQLite (internal page
 cache, not AsyncFileCached) by pinning pages which are not freed until the
 cursor is destroyed.

---
 fdbclient/ServerKnobs.cpp               |  1 +
 fdbclient/ServerKnobs.h                 |  1 +
 fdbserver/KeyValueStoreSQLite.actor.cpp | 10 +++++++++-
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index de3d6d5443..db9e99f950 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -290,6 +290,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SQLITE_CHUNK_SIZE_PAGES_SIM,                          1024 );  // 4MB
 	init( SQLITE_READER_THREADS,                                  64 );  // number of read threads
 	init( SQLITE_WRITE_WINDOW_SECONDS,                            -1 );
+	init( SQLITE_CURSOR_MAX_LIFETIME_BYTES,                      1e6 ); if( randomize && BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0;
 	init( SQLITE_WRITE_WINDOW_LIMIT,                              -1 );
 	if( randomize && BUGGIFY ) {
 		// Choose an window between .01 and 1.01 seconds.
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index cfe6c8d4d1..55f044998a 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -254,6 +254,7 @@ public:
 	int SQLITE_READER_THREADS;
 	int SQLITE_WRITE_WINDOW_LIMIT;
 	double SQLITE_WRITE_WINDOW_SECONDS;
+	int64_t SQLITE_CURSOR_MAX_LIFETIME_BYTES;
 
 	// KeyValueStoreSqlite spring cleaning
 	double SPRING_CLEANING_NO_ACTION_INTERVAL;
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index ea47ef0aa6..46bca480cf 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -706,6 +706,7 @@ struct RawCursor {
 	BtCursor* cursor;
 	KeyInfo keyInfo;
 	bool valid;
+	int64_t kvBytesRead = 0;
 
 	operator bool() const { return valid; }
 
@@ -1105,6 +1106,7 @@ struct RawCursor {
 			if (r == 0) {
 				Value result;
 				((ValueRef&)result) = decodeKVFragment(getEncodedRow(result.arena())).get().value;
+				kvBytesRead += key.size() + result.size();
 				return result;
 			}
 
@@ -1115,12 +1117,14 @@ struct RawCursor {
 			DefragmentingReader i(*this, m, true);
 			if (i.peek() == key) {
 				Optional<KeyValueRef> kv = i.getNext();
+				kvBytesRead += key.size() + kv.get().value.size();
 				return Value(kv.get().value, m);
 			}
 		} else if (r == 0) {
 			Value result;
 			KeyValueRef kv = decodeKV(getEncodedRow(result.arena()));
 			((ValueRef&)result) = kv.value;
+			kvBytesRead += key.size() + result.size();
 			return result;
 		}
 
@@ -1135,6 +1139,7 @@ struct RawCursor {
 			DefragmentingReader i(*this, m, getEncodedKVFragmentSize(key.size(), maxLength));
 			if (i.peek() == key) {
 				Optional<KeyValueRef> kv = i.getNext();
+				kvBytesRead += key.size() + kv.get().value.size();
 				return Value(kv.get().value, m);
 			}
 		} else if (!moveTo(key)) {
@@ -1145,6 +1150,7 @@ struct RawCursor {
 			int maxEncodedSize = getEncodedSize(key.size(), maxLength);
 			KeyValueRef kv = decodeKVPrefix(getEncodedRowPrefix(result.arena(), maxEncodedSize), maxLength);
 			((ValueRef&)result) = kv.value;
+			kvBytesRead += key.size() + result.size();
 			return result;
 		}
 		return Optional<Value>();
@@ -1221,6 +1227,8 @@ struct RawCursor {
 			ASSERT(result.size() > 0);
 			result.readThrough = result[result.size() - 1].key;
 		}
+		// AccumulatedBytes includes KeyValueRef overhead so subtract it
+		kvBytesRead += (accumulatedBytes - result.size() * sizeof(KeyValueRef));
 		return result;
 	}
 
@@ -1639,7 +1647,7 @@ private:
 
 		Reference<ReadCursor> getCursor() {
 			Reference<ReadCursor> cursor = *ppReadCursor;
-			if (!cursor) {
+			if (!cursor || cursor->get().kvBytesRead > SERVER_KNOBS->SQLITE_CURSOR_MAX_LIFETIME_BYTES) {
 				*ppReadCursor = cursor = makeReference<ReadCursor>();
 				cursor->init(conn);
 			}

From 36edfb8820fd885ad15d166f8d3aaf1c8993f86f Mon Sep 17 00:00:00 2001
From: Yao Xiao <87789492+yao-xiao-github@users.noreply.github.com>
Date: Mon, 7 Mar 2022 13:14:20 -0800
Subject: [PATCH 174/210] Update error type for timeouts. (#6503)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 1e966b8b9b..a129e1b613 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -1076,7 +1076,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.Before");
 			}
 			if (readBeginTime - a.startTime > readValueTimeout) {
-				TraceEvent(SevWarn, "RocksDBError")
+				TraceEvent(SevWarn, "KVSTimeout")
 				    .detail("Error", "Read value request timedout")
 				    .detail("Method", "ReadValueAction")
 				    .detail("Timeout value", readValueTimeout);
@@ -1153,7 +1153,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				                          "Reader.Before"); //.detail("TaskID", g_network->getCurrentTask());
 			}
 			if (readBeginTime - a.startTime > readValuePrefixTimeout) {
-				TraceEvent(SevWarn, "RocksDBError")
+				TraceEvent(SevWarn, "KVSTimeout")
 				    .detail("Error", "Read value prefix request timedout")
 				    .detail("Method", "ReadValuePrefixAction")
 				    .detail("Timeout value", readValuePrefixTimeout);
@@ -1224,7 +1224,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readRangeQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
 			}
 			if (readBeginTime - a.startTime > readRangeTimeout) {
-				TraceEvent(SevWarn, "RocksDBError")
+				TraceEvent(SevWarn, "KVSTimeout")
 				    .detail("Error", "Read range request timedout")
 				    .detail("Method", "ReadRangeAction")
 				    .detail("Timeout value", readRangeTimeout);
@@ -1255,7 +1255,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 						break;
 					}
 					if (timer_monotonic() - a.startTime > readRangeTimeout) {
-						TraceEvent(SevWarn, "RocksDBError")
+						TraceEvent(SevWarn, "KVSTimeout")
 						    .detail("Error", "Read range request timedout")
 						    .detail("Method", "ReadRangeAction")
 						    .detail("Timeout value", readRangeTimeout);
@@ -1286,7 +1286,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 						break;
 					}
 					if (timer_monotonic() - a.startTime > readRangeTimeout) {
-						TraceEvent(SevWarn, "RocksDBError")
+						TraceEvent(SevWarn, "KVSTimeout")
 						    .detail("Error", "Read range request timedout")
 						    .detail("Method", "ReadRangeAction")
 						    .detail("Timeout value", readRangeTimeout);

From a827e1cbd59f78e5845ef1cbfcb36b4f54d03e41 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Thu, 3 Mar 2022 13:55:43 -0800
Subject: [PATCH 175/210] Flush all output streams in flushAndExit

Also detect reading past the end of the infile for debug_determinism tracing
---
 contrib/debug_determinism/debug_determinism.cpp | 6 +++---
 flow/Platform.actor.cpp                         | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/contrib/debug_determinism/debug_determinism.cpp b/contrib/debug_determinism/debug_determinism.cpp
index 02982a8cbb..4d84ecc27a 100644
--- a/contrib/debug_determinism/debug_determinism.cpp
+++ b/contrib/debug_determinism/debug_determinism.cpp
@@ -40,11 +40,11 @@ extern "C" void __sanitizer_cov_trace_pc_guard(uint32_t* guard) {
 	if (!guard) {
 		return;
 	}
-	fwrite(guard, 1, sizeof(*guard), out);
+	fwrite(guard, sizeof(*guard), 1, out);
 	if (in) {
 		uint32_t theirs;
-		fread(&theirs, 1, sizeof(theirs), in);
-		if (*guard != theirs) {
+		auto read = fread(&theirs, sizeof(theirs), 1, in);
+		if (read != 1 || *guard != theirs) {
 			printf("Non-determinism detected\n");
 			loop_forever();
 		}
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index ffc1cf36ba..52a5b1504f 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -3231,6 +3231,10 @@ extern "C" void flushAndExit(int exitCode) {
 	flushTraceFileVoid();
 	fflush(stdout);
 	closeTraceFile();
+
+	// Flush all output streams. The original intent is to flush the outfile for contrib/debug_determinism.
+	fflush(nullptr);
+
 #ifdef USE_GCOV
 	__gcov_flush();
 #endif

From f39a53787f4c69441ea8291b55c4580e9b418b45 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Mon, 7 Mar 2022 16:33:12 -0800
Subject: [PATCH 176/210] Ignore changes that happen in the system key-space
 when evaluating whether a transaction should have conflicted in the conflict
 range test.

---
 fdbserver/workloads/ConflictRange.actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbserver/workloads/ConflictRange.actor.cpp b/fdbserver/workloads/ConflictRange.actor.cpp
index b6c9202f24..f4fa875f42 100644
--- a/fdbserver/workloads/ConflictRange.actor.cpp
+++ b/fdbserver/workloads/ConflictRange.actor.cpp
@@ -328,7 +328,8 @@ struct ConflictRangeWorkload : TestWorkload {
 
 					if (res.size() == originalResults.size()) {
 						for (int i = 0; i < res.size(); i++) {
-							if (res[i] != originalResults[i]) {
+							if (res[i] != originalResults[i] && !res[i].key.startsWith("\xff"_sr) &&
+							    !originalResults[i].key.startsWith("\xff"_sr)) {
 								TraceEvent(SevError, "ConflictRangeError")
 								    .detail("Info", "No conflict returned, however results do not match")
 								    .detail("Original",

From 5003a0d662a2ff203366f0c63f7ee1e8f371503d Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 8 Mar 2022 12:01:51 -0800
Subject: [PATCH 177/210] Remove unnecessary parameter from clampRangeToTenant

---
 fdbserver/storageserver.actor.cpp | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index aef837df52..1440562dbe 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -458,10 +458,7 @@ public:
 	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
 
 	Optional<TenantMapEntry> getTenantEntry(Version version, TenantInfo tenant);
-	KeyRangeRef clampRangeToTenant(KeyRangeRef range,
-	                               Optional<TenantMapEntry> tenantEntry,
-	                               Version version,
-	                               Arena& arena);
+	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantMapEntry> tenantEntry, Arena& arena);
 
 	class CurrentRunningFetchKeys {
 		std::unordered_map<UID, double> startTimeMap;
@@ -2468,7 +2465,6 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range,
                                               Optional<TenantMapEntry> tenantEntry,
-                                              Version version,
                                               Arena& arena) {
 	if (tenantEntry.present()) {
 		return KeyRangeRef(range.begin.startsWith(tenantEntry.get().prefix) ? range.begin : tenantEntry.get().prefix,
@@ -2660,7 +2656,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3167,7 +3163,7 @@ ACTOR Future<Void> getKeyValuesAndFlatMapQ(StorageServer* data, GetKeyValuesAndF
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3378,7 +3374,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			throw wrong_shard_server();
 		}
 
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.arena);
 
 		state int offset1 = 0;
 		state int offset2;
@@ -3551,7 +3547,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		state uint64_t changeCounter = data->shardChangeCounter;
 
 		KeyRange shard = getShardKeyRange(data, req.sel);
-		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.version, req.arena);
+		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.arena);
 
 		state int offset;
 		Key absoluteKey = wait(

From 24d0c158b4c00f8d972f7647b0c9734e256e3b3c Mon Sep 17 00:00:00 2001
From: Jon Fu <jon.fu@snowflake.com>
Date: Tue, 8 Mar 2022 16:22:09 -0500
Subject: [PATCH 178/210] Apply stricter condition for special keys check

---
 fdbserver/workloads/ConflictRange.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/ConflictRange.actor.cpp b/fdbserver/workloads/ConflictRange.actor.cpp
index f4fa875f42..29ce033743 100644
--- a/fdbserver/workloads/ConflictRange.actor.cpp
+++ b/fdbserver/workloads/ConflictRange.actor.cpp
@@ -328,8 +328,8 @@ struct ConflictRangeWorkload : TestWorkload {
 
 					if (res.size() == originalResults.size()) {
 						for (int i = 0; i < res.size(); i++) {
-							if (res[i] != originalResults[i] && !res[i].key.startsWith("\xff"_sr) &&
-							    !originalResults[i].key.startsWith("\xff"_sr)) {
+							if (res[i] != originalResults[i] &&
+							    !(res[i].key.startsWith("\xff"_sr) && originalResults[i].key.startsWith("\xff"_sr))) {
 								TraceEvent(SevError, "ConflictRangeError")
 								    .detail("Info", "No conflict returned, however results do not match")
 								    .detail("Original",

From 6f7adb04a5a7203e96fe1dd2cee6752a91609bc3 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 8 Mar 2022 13:33:56 -0800
Subject: [PATCH 179/210] Send a version along with
 GetKeyServerLocationsRequests to allow the proxy to reject missing tenants
 more quickly. Change the commit proxy tenant map to be unversioned.

---
 fdbclient/CommitProxyInterface.h              | 15 +++--
 fdbclient/NativeAPI.actor.cpp                 | 37 +++++++-----
 fdbserver/ApplyMetadataMutation.cpp           | 11 ++--
 fdbserver/CommitProxyServer.actor.cpp         | 56 +++++++++++--------
 fdbserver/ProxyCommitData.actor.h             |  2 +-
 fdbserver/storageserver.actor.cpp             |  4 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 15 +++--
 7 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h
index c50f6ba347..4e006284b5 100644
--- a/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/CommitProxyInterface.h
@@ -318,20 +318,27 @@ struct GetKeyServerLocationsRequest {
 	bool reverse;
 	ReplyPromise<GetKeyServerLocationsReply> reply;
 
-	GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
+	// This version is used to specify the minimum metadata version a proxy must have in order to declare that
+	// a tenant is not present. If the metadata version is lower, the proxy must wait in case the tenant gets
+	// created. If latestVersion is specified, then the proxy will wait until it is sure that it has received
+	// updates from other proxies before answering.
+	Version minTenantVersion;
+
+	GetKeyServerLocationsRequest() : limit(0), reverse(false), minTenantVersion(latestVersion) {}
 	GetKeyServerLocationsRequest(SpanID spanContext,
 	                             Optional<TenantNameRef> const& tenant,
 	                             KeyRef const& begin,
 	                             Optional<KeyRef> const& end,
 	                             int limit,
 	                             bool reverse,
+	                             Version minTenantVersion,
 	                             Arena const& arena)
-	  : arena(arena), spanContext(spanContext), tenant(tenant), begin(begin), end(end), limit(limit), reverse(reverse) {
-	}
+	  : arena(arena), spanContext(spanContext), tenant(tenant), begin(begin), end(end), limit(limit), reverse(reverse),
+	    minTenantVersion(minTenantVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, begin, end, limit, reverse, reply, spanContext, tenant, arena);
+		serializer(ar, begin, end, limit, reverse, reply, spanContext, tenant, minTenantVersion, arena);
 	}
 };
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 1f53b9290d..6008ebcf3f 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2545,13 +2545,18 @@ ACTOR Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation_intern
 		++cx->transactionKeyServerLocationRequests;
 		choose {
 			when(wait(cx->onProxiesChanged())) {}
-			when(
-			    GetKeyServerLocationsReply rep = wait(basicLoadBalance(
-			        cx->getCommitProxies(useProvisionalProxies),
-			        &CommitProxyInterface::getKeyServersLocations,
-			        GetKeyServerLocationsRequest(
-			            span.context, Optional<TenantNameRef>(), key, Optional<KeyRef>(), 100, isBackward, key.arena()),
-			        TaskPriority::DefaultPromiseEndpoint))) {
+			when(GetKeyServerLocationsReply rep =
+			         wait(basicLoadBalance(cx->getCommitProxies(useProvisionalProxies),
+			                               &CommitProxyInterface::getKeyServersLocations,
+			                               GetKeyServerLocationsRequest(span.context,
+			                                                            Optional<TenantNameRef>(),
+			                                                            key,
+			                                                            Optional<KeyRef>(),
+			                                                            100,
+			                                                            isBackward,
+			                                                            latestVersion,
+			                                                            key.arena()),
+			                               TaskPriority::DefaultPromiseEndpoint))) {
 				++cx->transactionKeyServerLocationRequestsCompleted;
 				if (debugID.present())
 					g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
@@ -2649,12 +2654,18 @@ ACTOR Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRa
 		++cx->transactionKeyServerLocationRequests;
 		choose {
 			when(wait(cx->onProxiesChanged())) {}
-			when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance(
-			         cx->getCommitProxies(useProvisionalProxies),
-			         &CommitProxyInterface::getKeyServersLocations,
-			         GetKeyServerLocationsRequest(
-			             span.context, Optional<TenantNameRef>(), keys.begin, keys.end, limit, reverse, keys.arena()),
-			         TaskPriority::DefaultPromiseEndpoint))) {
+			when(GetKeyServerLocationsReply _rep =
+			         wait(basicLoadBalance(cx->getCommitProxies(useProvisionalProxies),
+			                               &CommitProxyInterface::getKeyServersLocations,
+			                               GetKeyServerLocationsRequest(span.context,
+			                                                            Optional<TenantNameRef>(),
+			                                                            keys.begin,
+			                                                            keys.end,
+			                                                            limit,
+			                                                            reverse,
+			                                                            latestVersion,
+			                                                            keys.arena()),
+			                               TaskPriority::DefaultPromiseEndpoint))) {
 				++cx->transactionKeyServerLocationRequestsCompleted;
 				state GetKeyServerLocationsReply rep = _rep;
 				if (debugID.present())
diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp
index e0c421a22f..510f8ebc8e 100644
--- a/fdbserver/ApplyMetadataMutation.cpp
+++ b/fdbserver/ApplyMetadataMutation.cpp
@@ -118,7 +118,7 @@ private:
 	std::map<Tag, Version>* tag_popped = nullptr;
 	std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
 
-	TenantMap* tenantMap = nullptr;
+	std::map<TenantName, TenantMapEntry>* tenantMap = nullptr;
 
 	// true if the mutations were already written to the txnStateStore as part of recovery
 	bool initialCommit = false;
@@ -581,12 +581,11 @@ private:
 		if (m.param1.startsWith(tenantMapPrefix)) {
 			if (tenantMap) {
 				ASSERT(version != invalidVersion);
-				tenantMap->createNewVersion(version);
 				Standalone<StringRef> tenantName = m.param1.removePrefix(tenantMapPrefix);
 				TenantMapEntry tenantEntry = decodeTenantEntry(m.param2);
 
 				TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version);
-				tenantMap->insert(tenantName, tenantEntry);
+				(*tenantMap)[tenantName] = tenantEntry;
 			}
 
 			if (!initialCommit) {
@@ -939,7 +938,6 @@ private:
 		if (tenantMapKeys.intersects(range)) {
 			if (tenantMap) {
 				ASSERT(version != invalidVersion);
-				tenantMap->createNewVersion(version);
 
 				StringRef startTenant = std::max(range.begin, tenantMapPrefix).removePrefix(tenantMapPrefix);
 				StringRef endTenant = (range.end.startsWith(tenantMapPrefix) ? range.end : tenantMapKeys.end)
@@ -949,7 +947,10 @@ private:
 				    .detail("BeginTenant", startTenant)
 				    .detail("EndTenant", endTenant)
 				    .detail("Version", version);
-				tenantMap->erase(startTenant, endTenant);
+
+				auto startItr = tenantMap->lower_bound(startTenant);
+				auto endItr = tenantMap->lower_bound(endTenant);
+				tenantMap->erase(startItr, endItr);
 			}
 
 			if (!initialCommit) {
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 0ffbbc3a24..13f3729ef0 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -413,31 +413,27 @@ ACTOR static Future<ResolveTransactionBatchReply> trackResolutionMetrics(Referen
 ErrorOr<Optional<TenantMapEntry>> getTenantEntry(ProxyCommitData* commitData,
                                                  Optional<TenantNameRef> tenant,
                                                  Optional<int64_t> tenantId,
-                                                 Version commitVersion) {
+                                                 bool logOnFailure) {
 	if (tenant.present()) {
-		auto view = commitData->tenantMap.at(commitVersion);
-		auto itr = view.find(tenant.get());
-		if (itr == view.end()) {
-			if (commitVersion != latestVersion) {
-				TraceEvent(SevWarn, "CommitProxyUnknownTenant", commitData->dbgid)
-				    .detail("Tenant", tenant.get())
-				    .detail("Version", commitVersion);
+		auto itr = commitData->tenantMap.find(tenant.get());
+		if (itr == commitData->tenantMap.end()) {
+			if (logOnFailure) {
+				TraceEvent(SevWarn, "CommitProxyUnknownTenant", commitData->dbgid).detail("Tenant", tenant.get());
 			}
 
 			return unknown_tenant();
-		} else if (tenantId.present() && tenantId.get() != itr->id) {
-			if (commitVersion != latestVersion) {
+		} else if (tenantId.present() && tenantId.get() != itr->second.id) {
+			if (logOnFailure) {
 				TraceEvent(SevWarn, "CommitProxyTenantIdMismatch", commitData->dbgid)
 				    .detail("Tenant", tenant.get())
 				    .detail("TenantId", tenantId)
-				    .detail("ExistingId", itr->id)
-				    .detail("Version", commitVersion);
+				    .detail("ExistingId", itr->second.id);
 			}
 
 			return unknown_tenant();
 		}
 
-		return ErrorOr<Optional<TenantMapEntry>>(Optional<TenantMapEntry>(*itr));
+		return ErrorOr<Optional<TenantMapEntry>>(Optional<TenantMapEntry>(itr->second));
 	}
 
 	return Optional<TenantMapEntry>();
@@ -879,10 +875,8 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	int t;
 	for (t = 0; t < trs.size() && !self->forceRecovery; t++) {
 		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) {
-			ErrorOr<Optional<TenantMapEntry>> result = getTenantEntry(pProxyCommitData,
-			                                                          trs[t].tenantInfo.name.castTo<TenantNameRef>(),
-			                                                          trs[t].tenantInfo.tenantId,
-			                                                          self->commitVersion);
+			ErrorOr<Optional<TenantMapEntry>> result = getTenantEntry(
+			    pProxyCommitData, trs[t].tenantInfo.name.castTo<TenantNameRef>(), trs[t].tenantInfo.tenantId, true);
 
 			if (result.isError()) {
 				self->committed[t] = ConflictBatch::TransactionTenantFailure;
@@ -1539,22 +1533,36 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	wait(commitData->validState.getFuture());
 	wait(delay(0, TaskPriority::DefaultEndpoint));
 
-	state Version requestVersion = latestVersion;
 	state ErrorOr<Optional<TenantMapEntry>> tenantEntry;
+	state Version minTenantVersion =
+	    req.minTenantVersion == latestVersion ? commitData->stats.lastCommitVersionAssigned + 1 : req.minTenantVersion;
+
+	// If a large minTenantVersion is specified, we limit how long we wait for it to be available
+	state Future<Void> futureVersionDelay = minTenantVersion > commitData->stats.lastCommitVersionAssigned + 1
+	                                            ? delay(SERVER_KNOBS->FUTURE_VERSION_DELAY)
+	                                            : Never();
+
 	while (tenantEntry.isError()) {
+		bool finalQuery = commitData->version.get() >= minTenantVersion;
 		ErrorOr<Optional<TenantMapEntry>> _tenantEntry =
-		    getTenantEntry(commitData, req.tenant, Optional<int64_t>(), requestVersion);
+		    getTenantEntry(commitData, req.tenant, Optional<int64_t>(), finalQuery);
 		tenantEntry = _tenantEntry;
 
 		if (tenantEntry.isError()) {
-			if (requestVersion != latestVersion) {
+			if (finalQuery) {
 				req.reply.sendError(tenant_not_found());
 				return Void();
 			} else {
-				// If the tenant doesn't exist, wait until we are sure we've received updates from the other proxies to
-				// declare that it wasn't recently created.
-				wait(commitData->version.whenAtLeast(commitData->stats.lastCommitVersionAssigned + 1));
-				requestVersion = commitData->version.get();
+				choose {
+					// Wait until we are sure that we've received metadata updates through minTenantVersion
+					// If latestVersion is specified, this will wait until we have definitely received
+					// updates through the version at the time we received the request
+					when(wait(commitData->version.whenAtLeast(minTenantVersion))) {}
+					when(wait(futureVersionDelay)) {
+						req.reply.sendError(future_version());
+						return Void();
+					}
+				}
 			}
 		}
 	}
diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h
index a3518357a6..e4498f407a 100644
--- a/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/ProxyCommitData.actor.h
@@ -215,7 +215,7 @@ struct ProxyCommitData {
 	double lastMasterReset;
 	double lastResolverReset;
 
-	TenantMap tenantMap;
+	std::map<TenantName, TenantMapEntry> tenantMap;
 
 	// The tag related to a storage server rarely change, so we keep a vector of tags for each key range to be slightly
 	// more CPU efficient. When a tag related to a storage server does change, we empty out all of these vectors to
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 1440562dbe..93e3f38ffb 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2463,9 +2463,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 	return result;
 }
 
-KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range,
-                                              Optional<TenantMapEntry> tenantEntry,
-                                              Arena& arena) {
+KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range, Optional<TenantMapEntry> tenantEntry, Arena& arena) {
 	if (tenantEntry.present()) {
 		return KeyRangeRef(range.begin.startsWith(tenantEntry.get().prefix) ? range.begin : tenantEntry.get().prefix,
 		                   range.end.startsWith(tenantEntry.get().prefix)
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index c3d55d406e..5ee5e3c8f5 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -881,11 +881,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			for (int i = 0; i < commitProxyInfo->size(); i++)
 				keyServerLocationFutures.push_back(
 				    commitProxyInfo->get(i, &CommitProxyInterface::getKeyServersLocations)
-				        .getReplyUnlessFailedFor(
-				            GetKeyServerLocationsRequest(
-				                span.context, Optional<TenantNameRef>(), begin, end, limitKeyServers, false, Arena()),
-				            2,
-				            0));
+				        .getReplyUnlessFailedFor(GetKeyServerLocationsRequest(span.context,
+				                                                              Optional<TenantNameRef>(),
+				                                                              begin,
+				                                                              end,
+				                                                              limitKeyServers,
+				                                                              false,
+				                                                              latestVersion,
+				                                                              Arena()),
+				                                 2,
+				                                 0));
 
 			state bool keyServersInsertedForThisIteration = false;
 			choose {

From 6a2e54d955d4e07f247a88656386335895fcc42d Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 8 Mar 2022 13:45:29 -0800
Subject: [PATCH 180/210] Rename OPTIONAL to avoid windows macro conflict

---
 fdbclient/FDBTypes.h              | 9 +++------
 fdbclient/ManagementAPI.actor.cpp | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index ea53601792..48a8d6d3e3 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -30,10 +30,6 @@
 #include "flow/Arena.h"
 #include "flow/flow.h"
 
-#ifdef _WIN32
-#undef OPTIONAL
-#endif
-
 typedef int64_t Version;
 typedef uint64_t LogEpoch;
 typedef uint64_t Sequence;
@@ -1188,7 +1184,8 @@ struct StorageMigrationType {
 struct TenantMode {
 	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones
 	// just before END.
-	enum Mode { DISABLED = 0, OPTIONAL = 1, REQUIRED = 2, END = 3 };
+	// Note: OPTIONAL_TENANT is not named OPTIONAL because of a collision with a Windows macro.
+	enum Mode { DISABLED = 0, OPTIONAL_TENANT = 1, REQUIRED = 2, END = 3 };
 
 	TenantMode() : mode(DISABLED) {}
 	TenantMode(Mode mode) : mode(mode) {
@@ -1207,7 +1204,7 @@ struct TenantMode {
 		switch (mode) {
 		case DISABLED:
 			return "disabled";
-		case OPTIONAL:
+		case OPTIONAL_TENANT:
 			return "optional_experimental";
 		case REQUIRED:
 			return "required_experimental";
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index e88d2044bb..058b5ae0a8 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -179,7 +179,7 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			if (value == "disabled") {
 				tenantMode = TenantMode::DISABLED;
 			} else if (value == "optional_experimental") {
-				tenantMode = TenantMode::OPTIONAL;
+				tenantMode = TenantMode::OPTIONAL_TENANT;
 			} else if (value == "required_experimental") {
 				tenantMode = TenantMode::REQUIRED;
 			} else {

From d71e91eea5f7e08717c3a7bac2f72071d65bf4b5 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <aj.beamon@snowflake.com>
Date: Tue, 8 Mar 2022 14:35:22 -0800
Subject: [PATCH 181/210] Add a comment about limitsHint.

---
 fdbclient/SpecialKeySpace.actor.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h
index 13a49b055b..857c8c8ab0 100644
--- a/fdbclient/SpecialKeySpace.actor.h
+++ b/fdbclient/SpecialKeySpace.actor.h
@@ -36,6 +36,19 @@
 class SpecialKeyRangeReadImpl {
 public:
 	// Each derived class only needs to implement this simple version of getRange
+	//
+	// limitsHint can be used to reduce the amount of reading that the underlying
+	// implementation needs to do.
+	//
+	// NOTE: care needs to be taken when using limitsHint. If the range in question
+	// supports it, it is possible that some of the results may be removed when
+	// merged with mutations from the same transaction. If that happens, the final
+	// result may have fewer elements than the limit or even none at all if you didn't
+	// read the entire range.
+	//
+	// TODO: implement the range reading in a loop so that the underlying implementation
+	// can more naively fetch items up to the limit. If the merging deletes any entries,
+	// then the next set of entries can be read.
 	virtual Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                                     KeyRangeRef kr,
 	                                     GetRangeLimits limitsHint) const = 0;

From f230ac0e4164a4b40aa84eb4c962c7f8ba1c0806 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 9 Mar 2022 09:13:03 -0600
Subject: [PATCH 182/210] Fixing more merge issues

---
 fdbserver/BlobManager.actor.cpp                              | 2 +-
 fdbserver/BlobWorker.actor.cpp                               | 4 ++--
 fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp | 2 +-
 fdbserver/workloads/BlobGranuleVerifier.actor.cpp            | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index fae4c86d89..2fe9f034ca 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -1368,7 +1368,7 @@ ACTOR Future<Void> monitorBlobWorker(Reference<BlobManagerData> bmData, BlobWork
 				fmt::print("BM got unexpected error {0} monitoring BW {1}\n", e.name(), bwInterf.id().toString());
 			}
 			// TODO change back from SevError?
-			TraceEvent(SevError, "BWMonitoringFailed", bmData->id).detail("BlobWorkerID", bwInterf.id()).error(e);
+			TraceEvent(SevError, "BWMonitoringFailed", bmData->id).error(e).detail("BlobWorkerID", bwInterf.id());
 			throw e;
 		}
 	}
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index e578063174..15d83b6acd 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -726,9 +726,9 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 			wait(tr->onError(e));
 			retries++;
 			TraceEvent(SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id)
+			    .error(err)
 			    .detail("Granule", metadata->keyRange)
-			    .detail("Count", retries)
-			    .error(err);
+			    .detail("Count", retries);
 			bytesRead = 0;
 			lastReadVersion = readVersion;
 			// Pop change feed up to readVersion, because that data will be before the next snapshot
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index 8ecaeec694..c23b78758e 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -23,7 +23,7 @@
 #include <utility>
 #include <vector>
 
-#include "contrib/fmt-8.0.1/include/fmt/format.h"
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbclient/BlobGranuleReader.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index de77ecb34c..187cb90c9f 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -186,7 +186,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 					// reset the range and restart the read at a higher version
 					TraceEvent(SevDebug, "BGVFDBReadReset").detail("ReadVersion", v);
 					TEST(true); // BGV transaction reset
-					printf("Resetting BGV GRV {0} -> {1}\n", v, grv);
+					fmt::print("Resetting BGV GRV {0} -> {1}\n", v, grv);
 					first = true;
 					out = RangeResult();
 					currentRange = range;

From 4d909abbd1ef6c02bd482ea000fe4c75e96e5309 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 9 Mar 2022 09:21:45 -0600
Subject: [PATCH 183/210] re-enabling all storage engine types

---
 fdbserver/SimulatedCluster.actor.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index d5fe34cf34..b4b78a8f0d 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -1344,11 +1344,6 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		}
 	}
 
-	// TODO CHANGE BACK BEFORE MERGING!!
-	// Avoid memory storage engines for now to avoid extra memory pressure of change feeds, and redwood is ocasionally
-	// hitting a root too large issue with change feeds
-	storage_engine_type = 0;
-
 	switch (storage_engine_type) {
 	case 0: {
 		TEST(true); // Simulated cluster using ssd storage engine

From b21d0943b9c5199672acc5fbe43618d326fa1c33 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 9 Mar 2022 09:49:49 -0600
Subject: [PATCH 184/210] client-focused cleanup

---
 fdbclient/BlobWorkerInterface.h               |   8 +-
 fdbclient/DatabaseContext.h                   |   3 -
 fdbclient/NativeAPI.actor.cpp                 | 328 ++----------------
 fdbclient/StorageServerInterface.h            |   4 +-
 fdbrpc/fdbrpc.h                               |   4 -
 fdbserver/BlobManager.actor.cpp               |  18 +-
 fdbserver/BlobWorker.actor.cpp                |  16 +-
 fdbserver/storageserver.actor.cpp             |   4 +-
 fdbserver/worker.actor.cpp                    |  22 +-
 .../BlobGranuleCorrectnessWorkload.actor.cpp  |   3 +-
 .../workloads/BlobGranuleVerifier.actor.cpp   |  25 +-
 11 files changed, 65 insertions(+), 370 deletions(-)

diff --git a/fdbclient/BlobWorkerInterface.h b/fdbclient/BlobWorkerInterface.h
index 8be48404de..29318d6a1a 100644
--- a/fdbclient/BlobWorkerInterface.h
+++ b/fdbclient/BlobWorkerInterface.h
@@ -114,12 +114,10 @@ struct RevokeBlobRangeRequest {
 };
 
 /*
- * Continue: when a worker should continue handling a granule that was evaluated for a split
- * Reassign: when a new blob manager takes over, it sends Reassign requests to workers to redistribute granules
- * Normal: Neither continue nor reassign
+ * Continue: Blob worker should continue handling a granule that was evaluated for a split
+ * Normal: Blob worker should open the granule and start processing it
  */
-// TODO REMOVE reassign now!
-enum AssignRequestType { Normal = 0, Continue = 1, Reassign = 2 };
+enum AssignRequestType { Normal = 0, Continue = 1 };
 
 struct AssignBlobRangeRequest {
 	constexpr static FileIdentifier file_identifier = 905381;
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 890c1c8214..485617c102 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -181,8 +181,6 @@ struct ChangeFeedStorageData : ReferenceCounted<ChangeFeedStorageData> {
 	Promise<Void> destroyed;
 	UID interfToken;
 
-	bool debug = false; // TODO REMOVE
-
 	~ChangeFeedStorageData() { destroyed.send(Void()); }
 };
 
@@ -193,7 +191,6 @@ struct ChangeFeedData : ReferenceCounted<ChangeFeedData> {
 	Version getVersion();
 	Future<Void> whenAtLeast(Version version);
 
-	Key id; // TODO REMOVE eventually? for debugging
 	NotifiedVersion lastReturnedVersion;
 	std::vector<Reference<ChangeFeedStorageData>> storageData;
 	AsyncVar<int> notAtLatest;
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 482cc651fd..1963fb908c 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7427,46 +7427,19 @@ ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, Chan
 	state Promise<Void> destroyed = self->destroyed;
 	loop {
 		if (destroyed.isSet()) {
-			if (self->debug) {
-				fmt::print("CFSD {0}: destroyed\n", self->id.toString().substr(0, 4));
-			}
 			return Void();
 		}
 		if (self->version.get() < self->desired.get()) {
-			if (self->debug) {
-				fmt::print("CFSD {0}: update waiting {1} < {2}\n",
-				           self->id.toString().substr(0, 4),
-				           self->version.get(),
-				           self->desired.get());
-			}
 			wait(delay(CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME) || self->version.whenAtLeast(self->desired.get()));
 			if (destroyed.isSet()) {
-				if (self->debug) {
-					fmt::print("CFSD {0}: destroyed2\n", self->id.toString().substr(0, 4));
-				}
 				return Void();
 			}
-			if (self->debug) {
-				fmt::print("CFSD {0}: updated {1} < {2}\n",
-				           self->id.toString().substr(0, 4),
-				           self->version.get(),
-				           self->desired.get());
-			}
 			if (self->version.get() < self->desired.get()) {
-				if (self->debug) {
-					fmt::print("CFSD {0}: requesting {1}\n", self->id.toString().substr(0, 4), self->desired.get());
-				}
 				try {
 					ChangeFeedVersionUpdateReply rep = wait(brokenPromiseToNever(
 					    interf.changeFeedVersionUpdate.getReply(ChangeFeedVersionUpdateRequest(self->desired.get()))));
 
-					if (self->debug) {
-						fmt::print("CFSD {0}: got {1}\n", self->id.toString().substr(0, 4), rep.version);
-					}
 					if (rep.version > self->version.get()) {
-						if (self->debug) {
-							fmt::print("CFSD {0}: V={1} (req)\n", self->id.toString().substr(0, 4), rep.version);
-						}
 						self->version.set(rep.version);
 					}
 				} catch (Error& e) {
@@ -7481,9 +7454,6 @@ ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, Chan
 				}
 			}
 		} else {
-			if (self->debug) {
-				fmt::print("CFSD {0}: desired.WAL({1})\n", self->id.toString().substr(0, 4), self->version.get() + 1);
-			}
 			wait(self->desired.whenAtLeast(self->version.get() + 1));
 		}
 	}
@@ -7505,37 +7475,9 @@ Reference<ChangeFeedStorageData> DatabaseContext::getStorageData(StorageServerIn
 }
 
 Version ChangeFeedData::getVersion() {
-	// FIXME: add back in smarter version check later
-	/*if (notAtLatest.get() == 0 && mutations.isEmpty()) {
-	    Version v = storageData[0]->version.get();
-	    for (int i = 1; i < storageData.size(); i++) {
-	        if (storageData[i]->version.get() < v) {
-	            v = storageData[i]->version.get();
-	        }
-	    }
-	    return std::max(v, lastReturnedVersion.get());
-	}
-	*/
 	return lastReturnedVersion.get();
 }
 
-// TODO REMOVE when BG is correctness clean
-// To debug a waitLatest at wait_version returning early, set wait_version to the version, and start+end to a version
-// range that surrounds wait_version enough to figure out what's going on
-// DEBUG_CF_ID is optional
-#define DEBUG_CF_ID ""_sr
-#define DEBUG_CF_START_VERSION invalidVersion
-#define DEBUG_CF_END_VERSION invalidVersion
-#define DEBUG_CF_WAIT_VERSION invalidVersion
-#define DEBUG_CF_VERSION(cfId, v)                                                                                      \
-	DEBUG_CF_START_VERSION <= v&& v <= DEBUG_CF_END_VERSION && (""_sr == DEBUG_CF_ID || cfId.printable() == DEBUG_CF_ID)
-
-#define DEBUG_CF_VERSION_RANGE(cfId, vStart, vEnd)                                                                     \
-	DEBUG_CF_START_VERSION <= vEnd&& vStart <= DEBUG_CF_END_VERSION &&                                                 \
-	    (""_sr == DEBUG_CF_ID || cfId.printable() == DEBUG_CF_ID)
-
-#define DEBUG_CF_WAIT(cfId, v) DEBUG_CF_WAIT_VERSION == v && (""_sr == DEBUG_CF_ID || cfId.printable() == DEBUG_CF_ID)
-
 // This function is essentially bubbling the information about what has been processed from the server through the
 // change feed client. First it makes sure the server has returned all mutations up through the target version, the
 // native api has consumed and processed, them, and then the fdb client has consumed all of the mutations.
@@ -7548,9 +7490,6 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 		if (it->version.get() < version) {
 			waiting++;
 			if (version > it->desired.get()) {
-				if (DEBUG_CF_WAIT(self->id, version)) {
-					it->debug = true;
-				}
 				it->desired.set(version);
 				desired++;
 			}
@@ -7558,14 +7497,6 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 		}
 	}
 
-	if (DEBUG_CF_WAIT(self->id, version)) {
-		fmt::print("CFW {0})     WaitLatest: waiting for {1}/{2} ss ({3} < desired)\n",
-		           version,
-		           waiting,
-		           self->storageData.size(),
-		           desired);
-	}
-
 	wait(waitForAll(allAtLeast));
 
 	// then, wait on ss streams to have processed up through version
@@ -7576,10 +7507,6 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 		}
 	}
 
-	if (DEBUG_CF_WAIT(self->id, version)) {
-		fmt::print("CFW {0})     WaitLatest: waiting for {1} ss onEmpty\n", version, onEmpty.size());
-	}
-
 	if (onEmpty.size()) {
 		wait(waitForAll(onEmpty));
 	}
@@ -7592,42 +7519,20 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 	// done processing or we have up through the desired version
 	while (self->lastReturnedVersion.get() < self->maxSeenVersion && self->lastReturnedVersion.get() < version) {
 		Version target = std::min(self->maxSeenVersion, version);
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0})     WaitLatest: waiting merge lastReturned >= {1}\n", version, target);
-		}
 		wait(self->lastReturnedVersion.whenAtLeast(target));
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print(
-			    "CFW {0})     WaitLatest: got merge lastReturned {1}\n", version, self->lastReturnedVersion.get());
-		}
 	}
 
 	// then, wait for client to have consumed up through version
 	if (self->maxSeenVersion >= version) {
 		// merge cursor may have something buffered but has not yet sent it to self->mutations, just wait for
 		// lastReturnedVersion
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0})     WaitLatest: maxSeenVersion -> waiting lastReturned\n", version);
-		}
-
 		wait(self->lastReturnedVersion.whenAtLeast(version));
-
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0})     WaitLatest: maxSeenVersion -> got lastReturned\n", version);
-		}
 	} else {
 		// all mutations <= version are in self->mutations, wait for empty
 		while (!self->mutations.isEmpty()) {
-			if (DEBUG_CF_WAIT(self->id, version)) {
-				fmt::print("CFW {0})     WaitLatest: waiting for client onEmpty\n", version);
-			}
 			wait(self->mutations.onEmpty());
 			wait(delay(0));
 		}
-
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0})     WaitLatest: done\n", version);
-		}
 	}
 
 	return Void();
@@ -7635,28 +7540,14 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 
 ACTOR Future<Void> changeFeedWhenAtLatest(Reference<ChangeFeedData> self, Version version) {
 	if (version >= self->endVersion) {
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print(
-			    "CFW {0}) WhenAtLeast: After CF end version {1}, returning Never()\n", version, self->endVersion);
-		}
 		return Never();
 	}
-
-	if (DEBUG_CF_WAIT(self->id, version)) {
-		fmt::print("CFW {0}) WhenAtLeast: LR={1}\n", version, self->lastReturnedVersion.get());
-	}
 	if (version <= self->getVersion()) {
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0}) WhenAtLeast: Already done\n", version, self->lastReturnedVersion.get());
-		}
 		return Void();
 	}
 	state Future<Void> lastReturned = self->lastReturnedVersion.whenAtLeast(version);
 
 	loop {
-		if (DEBUG_CF_WAIT(self->id, version)) {
-			fmt::print("CFW {0})   WhenAtLeast: NotAtLatest={1}\n", version, self->notAtLatest.get());
-		}
 		// only allowed to use empty versions if you're caught up
 		Future<Void> waitEmptyVersion = (self->notAtLatest.get() == 0) ? changeFeedWaitLatest(self, version) : Never();
 		choose {
@@ -7667,10 +7558,6 @@ ACTOR Future<Void> changeFeedWhenAtLatest(Reference<ChangeFeedData> self, Versio
 		}
 	}
 
-	if (DEBUG_CF_VERSION(self->id, version)) {
-		fmt::print("CFLR (WAL): {0}\n", version);
-	}
-
 	if (self->lastReturnedVersion.get() < version) {
 		self->lastReturnedVersion.set(version);
 	}
@@ -7688,10 +7575,7 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
                                            Version begin,
                                            Version end,
                                            Reference<ChangeFeedData> feedData,
-                                           Reference<ChangeFeedStorageData> storageData,
-                                           int idx /* TODO REMOVE this param after correctness clean */,
-                                           KeyRange range /* TODO REMOVE this param after correctness clean */,
-                                           UID debugID /*TODO REMOVE this param after correctness clean*/) {
+                                           Reference<ChangeFeedStorageData> storageData) {
 
 	// calling lastReturnedVersion's callbacks could cause us to be cancelled
 	state Promise<Void> refresh = feedData->refresh;
@@ -7700,34 +7584,11 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 	try {
 		loop {
 			if (nextVersion >= end) {
-				if (DEBUG_CF_VERSION(feedData->id, end)) {
-					fmt::print("  single {0} {1} [{2} - {3}): sending EOS\n",
-					           idx,
-					           interf.id().toString().substr(0, 4),
-					           range.begin.printable(),
-					           range.end.printable());
-				}
 				results.sendError(end_of_stream());
 				return Void();
 			}
 			choose {
 				when(state ChangeFeedStreamReply rep = waitNext(replyStream.getFuture())) {
-					if (DEBUG_CF_VERSION_RANGE(
-					        feedData->id, rep.mutations.front().version, rep.mutations.back().version)) {
-						fmt::print("  single {0} {1} {2}: response {3} - {4} ({5}), atLatest={6}, rep.atLatest={7}, "
-						           "notAtLatest={8}, minSV={9}\n",
-						           idx,
-						           interf.id().toString().substr(0, 4),
-						           debugID.toString().substr(0, 8).c_str(),
-						           rep.mutations.front().version,
-						           rep.mutations.back().version,
-						           rep.mutations.size(),
-						           atLatestVersion ? "T" : "F",
-						           rep.atLatestVersion ? "T" : "F",
-						           feedData->notAtLatest.get(),
-						           rep.minStreamVersion);
-					}
-
 					// handle first empty mutation on stream establishment explicitly
 					if (nextVersion == begin && rep.mutations.size() == 1 && rep.mutations[0].mutations.size() == 0 &&
 					    rep.mutations[0].version == begin - 1) {
@@ -7744,39 +7605,9 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 					state int resultLoc = 0;
 					while (resultLoc < rep.mutations.size()) {
 						wait(results.onEmpty());
-						if (DEBUG_CF_VERSION(feedData->id, rep.mutations[resultLoc].version)) {
-							fmt::print("  single {0} {1} [{2} - {3}):   sending {4}/{5} {6} ({7})\n",
-							           idx,
-							           interf.id().toString().substr(0, 4),
-							           range.begin.printable(),
-							           range.end.printable(),
-							           resultLoc,
-							           rep.mutations.size(),
-							           rep.mutations[resultLoc].version,
-							           rep.mutations[resultLoc].mutations.size());
-						}
 						if (rep.mutations[resultLoc].version >= nextVersion) {
 							results.send(rep.mutations[resultLoc]);
 						} else {
-							// TODO REMOVE eventually, useful for debugging for now
-							if (!rep.mutations[resultLoc].mutations.empty()) {
-								fmt::print("non-empty mutations ({0}), but versions out of order from {1} for {2} cf "
-								           "{3}! mv={4}, nv={5}\n",
-								           rep.mutations.size(),
-								           interf.id().toString().substr(0, 4),
-								           idx,
-								           feedData->id.toString().substr(0, 6),
-								           rep.mutations[resultLoc].version,
-								           nextVersion);
-								for (auto& it : rep.mutations[resultLoc].mutations) {
-									if (it.type == MutationRef::SetValue) {
-										printf("  %s=", it.param1.printable().c_str());
-									} else {
-										printf(
-										    "  %s - %s", it.param1.printable().c_str(), it.param2.printable().c_str());
-									}
-								}
-							}
 							ASSERT(rep.mutations[resultLoc].mutations.empty());
 						}
 						resultLoc++;
@@ -7807,37 +7638,16 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 				when(wait(atLatestVersion && replyStream.isEmpty() && results.isEmpty()
 				              ? storageData->version.whenAtLeast(nextVersion)
 				              : Future<Void>(Never()))) {
-					if (DEBUG_CF_VERSION(feedData->id, nextVersion)) {
-						fmt::print("  single {0} {1}:   WAL {2}, sending empty {3})\n",
-						           idx,
-						           interf.id().toString().substr(0, 4),
-						           nextVersion,
-						           storageData->version.get());
-					}
 					MutationsAndVersionRef empty;
 					empty.version = storageData->version.get();
 					results.send(empty);
 					nextVersion = storageData->version.get() + 1;
 				}
 				when(wait(atLatestVersion && replyStream.isEmpty() && !results.isEmpty() ? results.onEmpty()
-				                                                                         : Future<Void>(Never()))) {
-					if (DEBUG_CF_VERSION(feedData->id, nextVersion)) {
-						fmt::print("  single {0} {1}:   got onEmpty\n", idx, interf.id().toString().substr(0, 4));
-					}
-				}
+				                                                                         : Future<Void>(Never()))) {}
 			}
 		}
 	} catch (Error& e) {
-		// TODO REMOVE eventually, useful for debugging for now
-		// if (DEBUG_CF_VERSION(feedData->id, nextVersion)) {
-		fmt::print("  single {0} {1} {2} [{3} - {4}): CFError {5}\n",
-		           idx,
-		           interf.id().toString().substr(0, 4),
-		           debugID.toString().substr(0, 8).c_str(),
-		           range.begin.printable(),
-		           range.end.printable(),
-		           e.name());
-		// }
 		if (e.code() == error_code_actor_cancelled) {
 			throw;
 		}
@@ -7846,14 +7656,11 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 	}
 }
 
-// TODO better name
-ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
-                             std::vector<std::pair<StorageServerInterface, KeyRange>> interfs,
-                             std::vector<MutationAndVersionStream> streams,
-                             Version* begin,
-                             Version end) {
-	// TODO REMOVE, a sanity check
-	state int eosCount = 0;
+ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> results,
+                                                 std::vector<std::pair<StorageServerInterface, KeyRange>> interfs,
+                                                 std::vector<MutationAndVersionStream> streams,
+                                                 Version* begin,
+                                                 Version end) {
 	state Promise<Void> refresh = results->refresh;
 	// with empty version handling in the partial cursor, all streams will always have a next element with version >=
 	// the minimum version of any stream's next element
@@ -7872,8 +7679,6 @@ ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
 
 	state int interfNum = 0;
 
-	// TODO minor optimization - could make this just a vector of indexes if each MutationAndVersionStream remembered
-	// its version index
 	state std::vector<MutationAndVersionStream> streamsUsed;
 	// initially, pull from all streams
 	for (auto& stream : streams) {
@@ -7893,13 +7698,10 @@ ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
 				if (e.code() != error_code_end_of_stream) {
 					throw e;
 				}
-				eosCount++;
 			}
 			interfNum++;
 		}
 
-		ASSERT(streams.size() - mutations.size() == eosCount);
-
 		if (mutations.empty()) {
 			throw end_of_stream();
 		}
@@ -7932,20 +7734,11 @@ ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
 		ASSERT(nextVersion >= *begin);
 
 		*begin = nextVersion + 1;
-		if (DEBUG_CF_VERSION(results->id, nextOut.back().version)) {
-			fmt::print("CFNA (merged): {0} ({1})\n", nextOut.back().version, nextOut.back().mutations.size());
-		}
 
 		// send mutations at nextVersion to the client
 		if (nextOut.back().mutations.empty()) {
 			ASSERT(results->mutations.isEmpty());
 		} else {
-			// TODO REMOVE, for debugging
-			if (nextOut.back().version <= results->lastReturnedVersion.get()) {
-				fmt::print("ERROR: merge cursor got mutations {0} <= lastReturnedVersion {1}",
-				           nextOut.back().version,
-				           results->lastReturnedVersion.get());
-			}
 			ASSERT(nextOut.back().version > results->lastReturnedVersion.get());
 
 			results->mutations.send(nextOut);
@@ -7953,9 +7746,6 @@ ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
 			wait(delay(0));
 		}
 
-		if (DEBUG_CF_VERSION(results->id, nextVersion)) {
-			fmt::print("CFLR (merged): {0}\n", nextVersion);
-		}
 		if (nextVersion > results->lastReturnedVersion.get()) {
 			results->lastReturnedVersion.set(nextVersion);
 		}
@@ -7964,12 +7754,7 @@ ACTOR Future<Void> doCFMerge(Reference<ChangeFeedData> results,
 
 ACTOR Future<Void> onCFErrors(std::vector<Future<Void>> onErrors) {
 	wait(waitForAny(onErrors));
-	// propagate error - TODO better way?
-	for (auto& f : onErrors) {
-		if (f.isError()) {
-			throw f.getError();
-		}
-	}
+	// TODO: just do waitForAny and remove onCFErrors if this doesn't trip
 	ASSERT(false);
 	return Void();
 }
@@ -7986,7 +7771,9 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	state std::vector<Future<Void>> onErrors(interfs.size());
 	state std::vector<MutationAndVersionStream> streams(interfs.size());
 
-	std::vector<UID> debugIDs;
+	TEST(interfs.size() > 10); // Large change feed merge cursor
+	TEST(interfs.size() > 100); // Very large change feed merge cursor
+
 	results->streams.clear();
 	for (auto& it : interfs) {
 		ChangeFeedStreamRequest req;
@@ -8000,9 +7787,6 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		if (replyBufferSize != -1 && req.replyBufferSize < CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES) {
 			req.replyBufferSize = CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES;
 		}
-		UID debugID = deterministicRandom()->randomUniqueID();
-		debugIDs.push_back(debugID);
-		req.debugID = debugID;
 		results->streams.push_back(it.first.changeFeedStream.getReplyStream(req));
 	}
 
@@ -8021,31 +7805,13 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	results->notAtLatest.set(interfs.size());
 	refresh.send(Void());
 
-	if (DEBUG_CF_START_VERSION != invalidVersion) {
-		fmt::print("Starting merge cursor for {0} @ {1} - {2}\n", interfs.size(), *begin, end);
-	}
 	for (int i = 0; i < interfs.size(); i++) {
 		onErrors[i] = results->streams[i].onError();
-		fetchers[i] = partialChangeFeedStream(interfs[i].first,
-		                                      streams[i].results,
-		                                      results->streams[i],
-		                                      *begin,
-		                                      end,
-		                                      results,
-		                                      results->storageData[i],
-		                                      i,
-		                                      interfs[i].second,
-		                                      debugIDs[i]);
-		if (DEBUG_CF_START_VERSION != invalidVersion) {
-			fmt::print("    [{0} - {1}): {2} {3}\n",
-			           interfs[i].second.begin.printable(),
-			           interfs[i].second.end.printable(),
-			           i,
-			           debugIDs[i].toString().substr(0, 8));
-		}
+		fetchers[i] = partialChangeFeedStream(
+		    interfs[i].first, streams[i].results, results->streams[i], *begin, end, results, results->storageData[i]);
 	}
 
-	wait(onCFErrors(onErrors) || doCFMerge(results, interfs, streams, begin, end));
+	wait(onCFErrors(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end));
 
 	return Void();
 }
@@ -8084,12 +7850,11 @@ ACTOR Future<KeyRange> getChangeFeedRange(Reference<DatabaseContext> db, Databas
 	}
 }
 
-ACTOR Future<Void> doSingleCFStream(KeyRange range,
-                                    Reference<ChangeFeedData> results,
-                                    Key rangeID,
-                                    Version* begin,
-                                    Version end,
-                                    UID debugID /*TODO REMOVE this parameter once BG is correctness clean*/) {
+ACTOR Future<Void> singleChangeFeedStreamInternal(KeyRange range,
+                                                  Reference<ChangeFeedData> results,
+                                                  Key rangeID,
+                                                  Version* begin,
+                                                  Version end) {
 
 	state Promise<Void> refresh = results->refresh;
 	ASSERT(results->streams.size() == 1);
@@ -8125,15 +7890,6 @@ ACTOR Future<Void> doSingleCFStream(KeyRange range,
 		if (anyMutations) {
 			// empty versions can come out of order, as we sometimes send explicit empty versions when restarting a
 			// stream. Anything with mutations should be strictly greater than lastReturnedVersion
-			if (feedReply.mutations.front().version <= results->lastReturnedVersion.get()) {
-				fmt::print("out of order mutation for CF {0} Req {1} from ({2}) {3}! {4} < {5}\n",
-				           rangeID.toString().substr(0, 6),
-				           debugID.toString().substr(0, 8),
-				           results->storageData.size(),
-				           results->storageData[0]->id.toString().substr(0, 4).c_str(),
-				           feedReply.mutations.front().version,
-				           results->lastReturnedVersion.get());
-			}
 			ASSERT(feedReply.mutations.front().version > results->lastReturnedVersion.get());
 
 			results->mutations.send(
@@ -8144,16 +7900,6 @@ ACTOR Future<Void> doSingleCFStream(KeyRange range,
 			wait(delay(0));
 		}
 
-		if (DEBUG_CF_VERSION(rangeID, feedReply.mutations.back().version)) {
-			fmt::print("CFLR (single) {0}: {1} ({2}), atLatest={3}, rep.atLatest={4}, notAtLatest={5}, minSV={6}\n",
-			           debugID.toString().substr(0, 8),
-			           feedReply.mutations.back().version,
-			           feedReply.mutations.size(),
-			           atLatest ? "T" : "F",
-			           feedReply.atLatestVersion ? "T" : "F",
-			           results->notAtLatest.get(),
-			           feedReply.minStreamVersion);
-		}
 		// check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we just
 		// skip to the next wait and get actor_cancelled
 		if (feedReply.mutations.back().version > results->lastReturnedVersion.get()) {
@@ -8165,11 +7911,6 @@ ACTOR Future<Void> doSingleCFStream(KeyRange range,
 			results->notAtLatest.set(0);
 		}
 		if (refresh.canBeSet() && feedReply.minStreamVersion > results->storageData[0]->version.get()) {
-			if (results->storageData[0]->debug) {
-				fmt::print("CFSD {0}: V={1} (CFLR)\n",
-				           results->storageData[0]->id.toString().substr(0, 4),
-				           results->storageData[0]->version.get());
-			}
 			results->storageData[0]->version.set(feedReply.minStreamVersion);
 		}
 	}
@@ -8186,14 +7927,12 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
                                           bool canReadPopped) {
 	state Database cx(db);
 	state ChangeFeedStreamRequest req;
-	state UID debugID = deterministicRandom()->randomUniqueID();
 	req.rangeID = rangeID;
 	req.begin = *begin;
 	req.end = end;
 	req.range = range;
 	req.canReadPopped = canReadPopped;
 	req.replyBufferSize = replyBufferSize;
-	req.debugID = debugID;
 
 	results->streams.clear();
 
@@ -8212,17 +7951,7 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 	results->notAtLatest.set(1);
 	refresh.send(Void());
 
-	if (DEBUG_CF_START_VERSION != invalidVersion) {
-		fmt::print("Starting single cursor {0} for [{1} - {2}) @ {3} - {4} from {5}\n",
-		           debugID.toString().substr(0, 8),
-		           range.begin.printable(),
-		           range.end.printable(),
-		           *begin,
-		           end,
-		           interf.id().toString().c_str());
-	}
-
-	wait(results->streams[0].onError() || doSingleCFStream(range, results, rangeID, begin, end, debugID));
+	wait(results->streams[0].onError() || singleChangeFeedStreamInternal(range, results, rangeID, begin, end));
 
 	return Void();
 }
@@ -8238,19 +7967,11 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 	state Database cx(db);
 	state Span span("NAPI:GetChangeFeedStream"_loc);
 
-	results->id = rangeID;
 	results->endVersion = end;
 
 	state double sleepWithBackoff = CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY;
 	state Version lastBeginVersion = invalidVersion;
 
-	/*printf("CFStream %s [%s - %s): [%lld - %lld]\n",
-	       rangeID.printable().substr(0, 6).c_str(),
-	       range.begin.printable().c_str(),
-	       range.end.printable().c_str(),
-	       begin,
-	       end);*/
-
 	loop {
 		state KeyRange keys;
 		try {
@@ -8319,17 +8040,19 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 					interfs.emplace_back(locations[i].second->getInterface(chosenLocations[i]),
 					                     locations[i].first & range);
 				}
+				TEST(true); // Change feed merge cursor
+				// TODO (jslocum): validate connectionFileChanged behavior
 				wait(
 				    mergeChangeFeedStream(db, interfs, results, rangeID, &begin, end, replyBufferSize, canReadPopped) ||
 				    cx->connectionFileChanged());
 			} else {
+				TEST(true); // Change feed single cursor
 				StorageServerInterface interf = locations[0].second->getInterface(chosenLocations[0]);
 				wait(singleChangeFeedStream(
 				         db, interf, range, results, rangeID, &begin, end, replyBufferSize, canReadPopped) ||
 				     cx->connectionFileChanged());
 			}
 		} catch (Error& e) {
-			fmt::print("CFNA error {}\n", e.name());
 			if (e.code() == error_code_actor_cancelled || e.code() == error_code_change_feed_popped) {
 				for (auto& it : results->storageData) {
 					if (it->debugGetReferenceCount() == 2) {
@@ -8339,6 +8062,7 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 				results->streams.clear();
 				results->storageData.clear();
 				if (e.code() == error_code_change_feed_popped) {
+					TEST(true); // getChangeFeedStreamActor got popped
 					results->mutations.sendError(e);
 					results->refresh.sendError(e);
 				} else {
@@ -8346,8 +8070,6 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 				}
 				throw;
 			}
-			// TODO REMOVE
-			// fmt::print("CFNA error {}\n", e.name());
 			if (results->notAtLatest.get() == 0) {
 				results->notAtLatest.set(1);
 			}
@@ -8355,8 +8077,6 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
 			    e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed ||
 			    e.code() == error_code_broken_promise) {
-				// TODO: add some exponential backoff (with a reasonable cap) when retrying if we didn't advance
-				// beginVersion at all before getting an error
 				db->changeFeedCache.erase(rangeID);
 				cx->invalidateCache(keys);
 				if (begin == lastBeginVersion) {
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 84bdfd95fe..cbe6937f2f 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -793,15 +793,13 @@ struct ChangeFeedStreamRequest {
 	KeyRange range;
 	int replyBufferSize = -1;
 	bool canReadPopped = true;
-	// TODO REMOVE once BG is correctness clean!! Useful for debugging
-	UID debugID;
 
 	ReplyPromiseStream<ChangeFeedStreamReply> reply;
 
 	ChangeFeedStreamRequest() {}
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeID, begin, end, range, reply, spanContext, replyBufferSize, canReadPopped, debugID, arena);
+		serializer(ar, rangeID, begin, end, range, reply, spanContext, replyBufferSize, canReadPopped, arena);
 	}
 };
 
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index c575788b1d..7490542138 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -449,9 +449,6 @@ public:
 		}
 	}
 
-	// TODO REMOVE
-	const void* debugAddr() const { return queue; }
-
 	template <class E>
 	void sendError(const E& exc) const {
 		if (queue->isRemoteEndpoint()) {
@@ -770,7 +767,6 @@ public:
 			} else {
 				Reference<Peer> peer =
 				    FlowTransport::transport().sendUnreliable(SerializeSource<T>(value), getEndpoint(), true);
-				// FIXME: defer sending the message until we know the connection is established
 				endStreamOnDisconnect(disc, p, getEndpoint(), peer);
 			}
 			return p;
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 2fe9f034ca..d21a1a9a79 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -470,7 +470,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			bmData->rangesToAssign.send(revokeOld);
 
 			// send assignment back to queue as is, clearing designated worker if present
-			// if we failed to send continue or reassign to the worker we thought owned the shard, it should be retried
+			// if we failed to send continue to the worker we thought owned the shard, it should be retried
 			// as a normal assign
 			ASSERT(assignment.assign.present());
 			assignment.assign.get().type = AssignRequestType::Normal;
@@ -952,14 +952,14 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				newLockSeqno = bmData->seqNo;
 				bmData->seqNo++;
 				if (!(bmData->epoch > ownerEpoch || (bmData->epoch == ownerEpoch && newLockSeqno > ownerSeqno))) {
-					printf("BM seqno for granule [%s - %s) out of order for lock! manager: (%lld, %lld), owner: %lld, "
-					       "%lld)\n",
-					       granuleRange.begin.printable().c_str(),
-					       granuleRange.end.printable().c_str(),
-					       bmData->epoch,
-					       newLockSeqno,
-					       ownerEpoch,
-					       ownerSeqno);
+					fmt::print("BM seqno for granule [{0} - {1}) out of order for lock! manager: ({2}, {3}), owner: "
+					           "({4}, {5}})\n",
+					           granuleRange.begin.printable().c_str(),
+					           granuleRange.end.printable().c_str(),
+					           bmData->epoch,
+					           newLockSeqno,
+					           ownerEpoch,
+					           ownerSeqno);
 				}
 				ASSERT(bmData->epoch > ownerEpoch || (bmData->epoch == ownerEpoch && newLockSeqno > ownerSeqno));
 			} else if (bmData->epoch == ownerEpoch && newLockSeqno < ownerSeqno) {
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 15d83b6acd..48b73727d8 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -1927,7 +1927,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					inFlightFiles[i].future.cancel();
 				}
 
-				// if we retry and re-open, we need to use a normal request (no continue or reassign) and update the
+				// if we retry and re-open, we need to use a normal request (no continue) and update the
 				// seqno
 				metadata->originalReq.managerEpoch = metadata->continueEpoch;
 				metadata->originalReq.managerSeqno = metadata->continueSeqno;
@@ -2787,20 +2787,6 @@ static bool changeBlobRange(Reference<BlobWorkerData> bwData,
 			fmt::print("thisAssignmentNewer={}\n", thisAssignmentNewer ? "true" : "false");
 		}
 
-		// if this granule already has it, and this was a special assignment (i.e. a new blob manager is
-		// trying to reassign granules), then just continue
-
-		// TODO this needs to also have the condition
-		if (active && assignType.get() == AssignRequestType::Reassign && r.value().activeMetadata.isValid() &&
-		    r.begin() == keyRange.begin && r.end() == keyRange.end) {
-			r.value().lastEpoch = epoch;
-			r.value().lastSeqno = seqno;
-			r.value().activeMetadata->continueEpoch = epoch;
-			r.value().activeMetadata->continueSeqno = seqno;
-			alreadyAssigned = true;
-			break;
-		}
-
 		if (BW_DEBUG) {
 			fmt::print("last: ({0}, {1}). now: ({2}, {3})\n", r.value().lastEpoch, r.value().lastSeqno, epoch, seqno);
 		}
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 5d62d9ff91..1b6fdaf4b3 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -8085,9 +8085,7 @@ ACTOR Future<Void> serveChangeFeedStreamRequests(StorageServer* self,
 		ChangeFeedStreamRequest req = waitNext(changeFeedStream);
 		// must notify change feed that its shard is moved away ASAP
 
-		// TODO CHANGE BACK after BG is correctness clean
-		// state UID streamUID = deterministicRandom()->randomUniqueID();
-		UID streamUID = req.debugID;
+		UID streamUID = deterministicRandom()->randomUniqueID();
 		self->actors.add(changeFeedStreamQ(self, req, streamUID) || stopChangeFeedOnMove(self, req, streamUID));
 	}
 }
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 0aedfb0397..ff6adde754 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1138,7 +1138,6 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
 		DUMPTOKEN(recruited.getKeyValueStoreType);
 		DUMPTOKEN(recruited.watchValue);
 		DUMPTOKEN(recruited.getKeyValuesStream);
-		DUMPTOKEN(recruited.getKeyValuesAndFlatMap);
 
 		prevStorageServer =
 		    storageServer(store, recruited, db, folder, Promise<Void>(), Reference<IClusterConnectionRecord>(nullptr));
@@ -1880,15 +1879,17 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 					TEST(true); // Recruited while already a blob manager.
 				} else {
-					// TODO: it'd be more optimal to halt the last manager here, but it will figure it out via the epoch
-					// check
-					// Also, not halting lets us handle the case here where the last BM had a higher epoch and somehow
-					// the epochs got out of order by a delayed initialize request. The one we start here will just halt
-					// on the lock check.
+					// TODO: it'd be more optimal to halt the last manager if present here, but it will figure it out
+					// via the epoch check
+					// Also, not halting lets us handle the case here where the last BM had a higher
+					// epoch and somehow the epochs got out of order by a delayed initialize request. The one we start
+					// here will just halt on the lock check.
 					myBMEpoch = req.epoch;
 					startRole(Role::BLOB_MANAGER, recruited.id(), interf.id());
 					DUMPTOKEN(recruited.waitFailure);
 					DUMPTOKEN(recruited.haltBlobManager);
+					DUMPTOKEN(recruited.haltBlobGranules);
+					DUMPTOKEN(recruited.blobManagerExclCheckReq);
 
 					Future<Void> blobManagerProcess = blobManager(recruited, dbInfo, req.epoch);
 					errorForwarders.add(forwardError(
@@ -2106,9 +2107,18 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					recruited.initEndpoints();
 					startRole(Role::BLOB_WORKER, recruited.id(), interf.id());
 
+					DUMPTOKEN(recruited.waitFailure);
+					DUMPTOKEN(recruited.blobGranuleFileRequest);
+					DUMPTOKEN(recruited.assignBlobRangeRequest);
+					DUMPTOKEN(recruited.revokeBlobRangeRequest);
+					DUMPTOKEN(recruited.granuleAssignmentsRequest);
+					DUMPTOKEN(recruited.granuleStatusStreamRequest);
+					DUMPTOKEN(recruited.haltBlobWorker);
+
 					ReplyPromise<InitializeBlobWorkerReply> blobWorkerReady = req.reply;
 					Future<Void> bw = blobWorker(recruited, blobWorkerReady, dbInfo);
 					errorForwarders.add(forwardError(errors, Role::BLOB_WORKER, recruited.id(), bw));
+
 				} else {
 					forwardPromise(req.reply, blobWorkerCache.get(req.reqId));
 				}
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index c23b78758e..352511fbb8 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -796,10 +796,9 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 	}
 
 	Future<Void> start(Database const& cx) override {
-		// TODO need to make thing that waits for granules to exist before ANY of the actors start!
-		// Then can reuse that for final checks or something?
 		clients.reserve(3 * directories.size());
 		for (auto& it : directories) {
+			// Wait for blob worker to initialize snapshot before starting test for that range
 			Future<Void> start = waitFirstSnapshot(this, cx, it, true);
 			clients.push_back(timeout(writeWorker(this, start, cx, it), testDuration, Void()));
 			clients.push_back(timeout(readWorker(this, start, cx, it), testDuration, Void()));
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index 187cb90c9f..04aa5e936d 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -327,7 +327,6 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 	// utility to prune <range> at pruneVersion=<version> with the <force> flag
 	ACTOR Future<Void> pruneAtVersion(Database cx, KeyRange range, Version version, bool force) {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state Version commitVersion = 0;
 		state Key pruneKey;
 		loop {
 			try {
@@ -343,19 +342,19 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 				Standalone<StringRef> vs = wait(fTrVs);
 				pruneKey = blobGranulePruneKeys.begin.withSuffix(vs);
 				if (BGV_DEBUG) {
-					printf("pruneAtVersion for range [%s-%s) at version %lld succeeded\n",
-					       range.begin.printable().c_str(),
-					       range.end.printable().c_str(),
-					       version);
+					fmt::print("pruneAtVersion for range [{0} - {1}) at version {2} succeeded\n",
+					           range.begin.printable(),
+					           range.end.printable(),
+					           version);
 				}
 				break;
 			} catch (Error& e) {
 				if (BGV_DEBUG) {
-					printf("pruneAtVersion for range [%s-%s) at version %lld encountered error %s\n",
-					       range.begin.printable().c_str(),
-					       range.end.printable().c_str(),
-					       version,
-					       e.name());
+					fmt::print("pruneAtVersion for range [{0} - {1}) at version {2} encountered error {3}\n",
+					           range.begin.printable(),
+					           range.end.printable(),
+					           version,
+					           e.name());
 				}
 				wait(tr->onError(e));
 			}
@@ -487,9 +486,6 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 								ASSERT(e.code() == error_code_blob_granule_transaction_too_old);
 							}
 						}
-
-						// TODO: read at some version older than pruneVersion and make sure you get txn_too_old
-						// To achieve this, the BWs are going to have to recognize latest prune versions per granules
 					} catch (Error& e) {
 						if (e.code() == error_code_blob_granule_transaction_too_old && oldRead.v >= dbgPruneVersion) {
 							self->timeTravelTooOld++;
@@ -556,8 +552,6 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 				Version readVersion = wait(tr->getReadVersion());
 				return readVersion;
 			} catch (Error& e) {
-				// TODO REMOVE print
-				printf("BGV GRV got error %s\n", e.name());
 				wait(tr->onError(e));
 			}
 		}
@@ -566,7 +560,6 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 	ACTOR Future<bool> _check(Database cx, BlobGranuleVerifierWorkload* self) {
 		// check error counts, and do an availability check at the end
 
-		// TODO need to have retry loop for getReadVersion, it's throwing tag throttled for some reason?
 		state Transaction tr(cx);
 		state Version readVersion = wait(self->doGrv(&tr));
 		state Version startReadVersion = readVersion;

From 9dbb6d216e75a96752f29f13345b97a96a743b8f Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 9 Mar 2022 10:39:04 -0600
Subject: [PATCH 185/210] SS-focused cleanup

---
 fdbserver/BlobManager.actor.cpp   |   2 +-
 fdbserver/BlobWorker.actor.cpp    |   2 +-
 fdbserver/storageserver.actor.cpp | 388 +++++++-----------------------
 3 files changed, 86 insertions(+), 306 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index d21a1a9a79..7c16df3c11 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -43,7 +43,7 @@
 #include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // has to be last include
 
-#define BM_DEBUG true
+#define BM_DEBUG false
 
 // TODO add comments + documentation
 void handleClientBlobRange(KeyRangeMap<bool>* knownBlobRanges,
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 48b73727d8..f8afcf0e73 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -45,7 +45,7 @@
 #include "flow/Trace.h"
 #include "flow/actorcompiler.h" // has to be last include
 
-#define BW_DEBUG true
+#define BW_DEBUG false
 #define BW_REQUEST_DEBUG false
 
 struct GranuleStartState {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 1b6fdaf4b3..b35fe3f611 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -23,6 +23,7 @@
 #include <type_traits>
 #include <unordered_map>
 
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/LoadBalance.h"
 #include "flow/ActorCollection.h"
@@ -392,8 +393,8 @@ struct ChangeFeedInfo : ReferenceCounted<ChangeFeedInfo> {
 
 	void moved(KeyRange range) {
 		auto toTrigger = moveTriggers.intersectingRanges(range);
-		for (auto triggerRange : toTrigger) {
-			for (auto triggerStream : triggerRange.cvalue()) {
+		for (auto& triggerRange : toTrigger) {
+			for (auto& triggerStream : triggerRange.cvalue()) {
 				if (triggerStream.second.canBeSet()) {
 					triggerStream.second.send(Void());
 				}
@@ -410,7 +411,7 @@ struct ChangeFeedInfo : ReferenceCounted<ChangeFeedInfo> {
 			ASSERT(streamToRemove != triggerRange->cvalue().end());
 			triggerRange->value().erase(streamToRemove);
 		}
-		// TODO: could clean up on
+		// TODO: may be more cleanup possible here
 	}
 };
 
@@ -1203,7 +1204,7 @@ public:
 		auto& clientVersions = changeFeedClientVersions[addr];
 		Version minVersion = version.get();
 		for (auto& it : clientVersions) {
-			// printf("Blocked client %s @ %lld\n", it.first.toString().substr(0, 8).c_str(), it.second);
+			// fmt::print("Blocked client {0} @ {1}\n", it.first.toString().substr(0, 8), it.second);
 			minVersion = std::min(minVersion, it.second);
 		}
 		return minVersion;
@@ -1892,16 +1893,11 @@ MutationsAndVersionRef filterMutations(Arena& arena,
 	return m;
 }
 
-// TODO REMOVE!!! when BG is correctness clean
-#define DEBUG_SS_ID ""_sr
-#define DEBUG_SS_CF_ID ""_sr
-#define DEBUG_SS_STREAM_ID ""_sr
-#define DEBUG_SS_CF_BEGIN_VERSION invalidVersion
-#define DEBUG_SS_CFM(ssId, cfId, streamId, v)                                                                          \
-	((ssId.toString().substr(0, 4) == DEBUG_SS_ID && cfId.printable().substr(0, 6) == DEBUG_SS_CF_ID &&                \
-	  (v >= DEBUG_SS_CF_BEGIN_VERSION || latestVersion == DEBUG_SS_CF_BEGIN_VERSION)) ||                               \
-	 (streamId.toString().substr(0, 8) == DEBUG_SS_STREAM_ID))
+// set this for VERY verbose logs on change feed SS reads
+#define DEBUG_CF_TRACE false
 
+// To easily find if a change feed read missed data. Set the CF to the feedId, the key to the missing key, and the
+// version to the version the mutation is missing at.
 #define DO_DEBUG_CF_MISSING false
 #define DEBUG_CF_MISSING_CF ""_sr
 #define DEBUG_CF_MISSING_KEY ""_sr
@@ -1915,25 +1911,15 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
                                                                             ChangeFeedStreamRequest req,
                                                                             bool inverted,
                                                                             bool atLatest,
-                                                                            UID streamUID
-                                                                            /*TODO REMOVE*/) {
+                                                                            UID streamUID /* for debugging */) {
 	state ChangeFeedStreamReply reply;
 	state ChangeFeedStreamReply memoryReply;
 	state int remainingLimitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 	state int remainingDurableBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 	state Version startVersion = data->version.get();
 
-	if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-		printf("CFM: SS %s CF %s: SQ %s [%s - %s) %lld - %lld\n",
-		       data->thisServerID.toString().substr(0, 4).c_str(),
-		       req.rangeID.printable().substr(0, 6).c_str(),
-		       streamUID.toString().substr(0, 8).c_str(),
-		       req.range.begin.printable().c_str(),
-		       req.range.end.printable().c_str(),
-		       req.begin,
-		       req.end);
-		// TODO REMOVE
-		TraceEvent(SevDebug, "ChangeFeedMutations", data->thisServerID)
+	if (DEBUG_CF_TRACE) {
+		TraceEvent(SevDebug, "TraceChangeFeedMutationsBegin", data->thisServerID)
 		    .detail("FeedID", req.rangeID)
 		    .detail("StreamUID", streamUID)
 		    .detail("Range", req.range)
@@ -1961,14 +1947,6 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 
 	state Reference<ChangeFeedInfo> feedInfo = feed->second;
 
-	if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-		printf("CFM: SS %s CF %s:   got version %lld >= %lld\n",
-		       data->thisServerID.toString().substr(0, 4).c_str(),
-		       req.rangeID.printable().substr(0, 6).c_str(),
-		       data->version.get(),
-		       req.begin);
-	}
-
 	// We must copy the mutationDeque when fetching the durable bytes in case mutations are popped from memory while
 	// waiting for the results
 	state Version dequeVersion = data->version.get();
@@ -1976,22 +1954,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	state Version emptyVersion = feedInfo->emptyVersion;
 	Version fetchStorageVersion = std::max(feedInfo->fetchVersion, feedInfo->durableFetchVersion.get());
 
-	if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-		printf("CFM: SS %s CF %s: SQ %s   atLatest=%s, dequeVersion=%lld, emptyVersion=%lld, storageVersion=%lld, "
-		       "durableVersion=%lld, "
-		       "fetchStorageVersion=%lld (%lld, %lld)\n",
-		       data->thisServerID.toString().substr(0, 4).c_str(),
-		       req.rangeID.printable().substr(0, 6).c_str(),
-		       streamUID.toString().substr(0, 8).c_str(),
-		       atLatest ? "T" : "F",
-		       dequeVersion,
-		       feedInfo->emptyVersion,
-		       feedInfo->storageVersion,
-		       feedInfo->durableVersion,
-		       fetchStorageVersion,
-		       feedInfo->fetchVersion,
-		       feedInfo->durableFetchVersion.get());
-		TraceEvent(SevDebug, "ChangeFeedMutationsDetails", data->thisServerID)
+	if (DEBUG_CF_TRACE) {
+		TraceEvent(SevDebug, "TraceChangeFeedMutationsDetails", data->thisServerID)
 		    .detail("FeedID", req.rangeID)
 		    .detail("StreamUID", streamUID)
 		    .detail("Range", req.range)
@@ -2022,14 +1986,6 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 				}
 			}
 		}
-		if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-			printf("CFM: SS %s CF %s:     got %lld - %lld (%d) from memory\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       memoryReply.mutations.empty() ? invalidVersion : memoryReply.mutations.front().version,
-			       memoryReply.mutations.empty() ? invalidVersion : memoryReply.mutations.back().version,
-			       memoryReply.mutations.size());
-		}
 	}
 
 	state bool readDurable = feedInfo->durableVersion != invalidVersion && req.begin <= feedInfo->durableVersion;
@@ -2044,23 +2000,10 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			ASSERT(req.begin <= feedInfo->fetchVersion);
 			TEST(true); // getChangeFeedMutations before fetched data durable
 
-			if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-				printf("CFM: SS %s CF %s:     waiting on fetch durable up to %lld\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       req.rangeID.printable().substr(0, 6).c_str(),
-				       feedInfo->fetchVersion);
-			}
-
 			// Wait for next commit to write pending feed data to storage
 			wait(feedInfo->durableFetchVersion.whenAtLeast(feedInfo->fetchVersion));
 			// To let update storage finish
 			wait(delay(0));
-			if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-				printf("CFM: SS %s CF %s:     got fetch durable up to %lld\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       req.rangeID.printable().substr(0, 6).c_str(),
-				       feedInfo->durableFetchVersion.get());
-			}
 		}
 		RangeResult res = wait(
 		    data->storage.readRange(KeyRangeRef(changeFeedDurableKey(req.rangeID, std::max(req.begin, emptyVersion)),
@@ -2097,23 +2040,22 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 					memoryVerifyIdx++;
 					continue;
 				} else {
-					printf("ERROR: SS %s CF %s SQ %s has mutation at %lld in memory but not on disk (next disk is "
-					       "%lld) "
-					       "(emptyVersion=%lld, emptyBefore=%lld)!\n",
-					       data->thisServerID.toString().substr(0, 4).c_str(),
-					       req.rangeID.printable().substr(0, 6).c_str(),
-					       streamUID.toString().substr(0, 8).c_str(),
-					       memoryReply.mutations[memoryVerifyIdx].version,
-					       version,
-					       feedInfo->emptyVersion,
-					       emptyVersion);
+					fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but not on disk (next disk "
+					           "is {4}) (emptyVersion={5}, emptyBefore={6})!\n",
+					           data->thisServerID.toString().substr(0, 4),
+					           req.rangeID.printable().substr(0, 6),
+					           streamUID.toString().substr(0, 8),
+					           memoryReply.mutations[memoryVerifyIdx].version,
+					           version,
+					           feedInfo->emptyVersion,
+					           emptyVersion);
 
-					printf("  Memory: (%d)\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
+					fmt::print("  Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
 					for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) {
 						if (it.type == MutationRef::SetValue) {
-							printf("    %s=\n", it.param1.printable().c_str());
+							fmt::print("    {}=\n", it.param1.printable());
 						} else {
-							printf("    %s - %s\n", it.param1.printable().c_str(), it.param2.printable().c_str());
+							fmt::print("    {} - {}\n", it.param1.printable(), it.param2.printable());
 						}
 					}
 					ASSERT(false);
@@ -2128,25 +2070,24 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 
 				if (memoryVerifyIdx < memoryReply.mutations.size() &&
 				    version == memoryReply.mutations[memoryVerifyIdx].version) {
-					// TODO: we could do some validation here too, but it's complicated because clears can get split
+					// We could do validation of mutations here too, but it's complicated because clears can get split
 					// and stuff
 					memoryVerifyIdx++;
 				}
 			} else if (memoryVerifyIdx < memoryReply.mutations.size() &&
 			           version == memoryReply.mutations[memoryVerifyIdx].version) {
-				// TODO REMOVE debugging eventually
-				printf("ERROR: SS %s CF %s SQ %s has mutation at %lld in memory but all filtered out on disk!\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       req.rangeID.printable().substr(0, 6).c_str(),
-				       streamUID.toString().substr(0, 8).c_str(),
-				       version);
+				fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but all filtered out on disk!\n",
+				           data->thisServerID.toString().substr(0, 4),
+				           req.rangeID.printable().substr(0, 6),
+				           streamUID.toString().substr(0, 8),
+				           version);
 
-				printf("  Memory: (%d)\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
+				fmt::print("  Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
 				for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) {
 					if (it.type == MutationRef::SetValue) {
-						printf("    %s=\n", it.param1.printable().c_str());
+						fmt::print("    {}=\n", it.param1.printable().c_str());
 					} else {
-						printf("    %s - %s\n", it.param1.printable().c_str(), it.param2.printable().c_str());
+						fmt::print("    {} - {}\n", it.param1.printable().c_str(), it.param2.printable().c_str());
 					}
 				}
 				ASSERT(false);
@@ -2158,14 +2099,6 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			lastVersion = version;
 			lastKnownCommitted = knownCommittedVersion;
 		}
-		if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-			printf("CFM: SS %s CF %s:     got %lld - %lld (%d) from disk\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       reply.mutations.empty() ? invalidVersion : reply.mutations.front().version,
-			       reply.mutations.empty() ? invalidVersion : reply.mutations.back().version,
-			       reply.mutations.size());
-		}
 		if (remainingDurableBytes > 0) {
 			reply.arena.dependsOn(memoryReply.arena);
 			auto it = memoryReply.mutations.begin();
@@ -2178,21 +2111,9 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			// If still empty, that means disk results were filtered out, but skipped all memory results. Add an empty,
 			// either the last version from disk
 			if (reply.mutations.empty() && res.size()) {
-				if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-					printf("CFM: SS %s CF %s:     adding empty from disk and memory %lld\n",
-					       data->thisServerID.toString().substr(0, 4).c_str(),
-					       req.rangeID.printable().substr(0, 6).c_str(),
-					       lastVersion);
-				}
 				reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 			}
 		} else if (reply.mutations.empty() || reply.mutations.back().version < lastVersion) {
-			if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-				printf("CFM: SS %s CF %s:     adding empty from disk %lld\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       req.rangeID.printable().substr(0, 6).c_str(),
-				       lastVersion);
-			}
 			reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 		}
 	} else {
@@ -2203,12 +2124,6 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	Version finalVersion = std::min(req.end - 1, dequeVersion);
 	if ((reply.mutations.empty() || reply.mutations.back().version < finalVersion) && remainingLimitBytes > 0 &&
 	    remainingDurableBytes > 0) {
-		if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-			printf("CFM: SS %s CF %s:     adding empty %lld\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       finalVersion);
-		}
 		reply.mutations.push_back(
 		    reply.arena, MutationsAndVersionRef(finalVersion, finalVersion == dequeVersion ? dequeKnownCommit : 0));
 		// if we add empty mutation after the last thing in memory, and didn't read from disk, gotAll is true
@@ -2229,16 +2144,15 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		TEST(ok); // feed popped while valid read waiting
 		TEST(!ok); // feed popped while invalid read waiting
 		if (!ok) {
-			printf("SS %s:   CF %s SQ %s popped after read! req.begin=%lld, emptyVersion=%lld, emptyBeforeRead=%lld, "
-			       "atLatest=%s, minVersionSent=%lld\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       streamUID.toString().substr(0, 8).c_str(),
-			       req.begin,
-			       feedInfo->emptyVersion,
-			       emptyVersion,
-			       atLatest ? "T" : "F",
-			       minVersion);
+			TraceEvent("ChangeFeedMutationsPopped", data->thisServerID)
+			    .detail("FeedID", req.rangeID)
+			    .detail("StreamUID", streamUID)
+			    .detail("Range", req.range)
+			    .detail("Begin", req.begin)
+			    .detail("End", req.end)
+			    .detail("EmptyVersion", feedInfo->emptyVersion)
+			    .detail("AtLatest", atLatest)
+			    .detail("MinVersionSent", minVersion);
 			throw change_feed_popped();
 		}
 	}
@@ -2246,7 +2160,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	if (MUTATION_TRACKING_ENABLED) {
 		for (auto& mutations : reply.mutations) {
 			for (auto& m : mutations.mutations) {
-				DEBUG_MUTATION("ChangeFeedRead", mutations.version, m, data->thisServerID)
+				DEBUG_MUTATION("ChangeFeedSSRead", mutations.version, m, data->thisServerID)
 				    .detail("ChangeFeedID", req.rangeID)
 				    .detail("StreamUID", streamUID)
 				    .detail("ReqBegin", req.begin)
@@ -2256,14 +2170,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		}
 	}
 
-	if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-		printf("CFM: SS %s CF %s:   result %lld - %lld (%d)\n",
-		       data->thisServerID.toString().substr(0, 4).c_str(),
-		       req.rangeID.printable().substr(0, 6).c_str(),
-		       reply.mutations.empty() ? invalidVersion : reply.mutations.front().version,
-		       reply.mutations.empty() ? invalidVersion : reply.mutations.back().version,
-		       reply.mutations.size());
-		// TODO REMOVE
+	if (DEBUG_CF_TRACE) {
 		TraceEvent(SevDebug, "ChangeFeedMutationsDone", data->thisServerID)
 		    .detail("FeedID", req.rangeID)
 		    .detail("StreamUID", streamUID)
@@ -2292,36 +2199,36 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			}
 		}
 		if (!foundVersion || !foundKey) {
-			printf("ERROR: SS %s CF %s SQ %s missing %s @ %lld from request for [%s - %s) %lld - %lld\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       streamUID.toString().substr(0, 8).c_str(),
-			       foundVersion ? "key" : "version",
-			       DEBUG_CF_MISSING_VERSION,
-			       req.range.begin.printable().c_str(),
-			       req.range.end.printable().c_str(),
-			       req.begin,
-			       req.end);
-			printf("ERROR: %d versions in response %lld - %lld:\n",
-			       reply.mutations.size(),
-			       reply.mutations.front().version,
-			       reply.mutations.back().version);
+			fmt::print("ERROR: SS {0} CF {1} SQ {2} missing {3} @ {4} from request for [{5} - {6}) {7} - {8}\n",
+			           data->thisServerID.toString().substr(0, 4),
+			           req.rangeID.printable().substr(0, 6),
+			           streamUID.toString().substr(0, 8),
+			           foundVersion ? "key" : "version",
+			           DEBUG_CF_MISSING_VERSION,
+			           req.range.begin.printable(),
+			           req.range.end.printable(),
+			           req.begin,
+			           req.end);
+			fmt::print("ERROR: {0} versions in response {1} - {2}:\n",
+			           reply.mutations.size(),
+			           reply.mutations.front().version,
+			           reply.mutations.back().version);
 			for (auto& it : reply.mutations) {
-				printf("ERROR:    %lld (%d)%s\n",
-				       it.version,
-				       it.mutations.size(),
-				       it.version == DEBUG_CF_MISSING_VERSION ? "<-------" : "");
+				fmt::print("ERROR:    {0} ({1}){2}\n",
+				           it.version,
+				           it.mutations.size(),
+				           it.version == DEBUG_CF_MISSING_VERSION ? "<-------" : "");
 			}
 		} else {
-			printf("DBG: SS %s CF %s SQ %s correct @ %lld from request for [%s - %s) %lld - %lld\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       streamUID.toString().substr(0, 8).c_str(),
-			       DEBUG_CF_MISSING_VERSION,
-			       req.range.begin.printable().c_str(),
-			       req.range.end.printable().c_str(),
-			       req.begin,
-			       req.end);
+			fmt::print("DBG: SS {0} CF {1} SQ {2} correct @ {3} from request for [{4} - {5}) {6} - {7}\n",
+			           data->thisServerID.toString().substr(0, 4),
+			           req.rangeID.printable().substr(0, 6),
+			           streamUID.toString().substr(0, 8),
+			           DEBUG_CF_MISSING_VERSION,
+			           req.range.begin.printable(),
+			           req.range.end.printable(),
+			           req.begin,
+			           req.end);
 		}
 	}
 
@@ -2391,7 +2298,6 @@ ACTOR Future<Void> stopChangeFeedOnMove(StorageServer* data, ChangeFeedStreamReq
 		}
 		return Void();
 	}
-	printf("CFSQ %s Moved! %lld - %lld. sending WSS\n", streamUID.toString().substr(0, 8).c_str(), req.begin, req.end);
 	// DO NOT call req.reply.onReady before sending - we need to propagate this error through regardless of how far
 	// behind client is
 	req.reply.sendError(wrong_shard_server());
@@ -2406,22 +2312,20 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 	if (req.replyBufferSize <= 0) {
 		req.reply.setByteLimit(SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES);
 	} else {
-		req.reply.setByteLimit(req.replyBufferSize);
+		req.reply.setByteLimit(std::min((int64_t)req.replyBufferSize, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES));
 	}
 
 	wait(delay(0, TaskPriority::DefaultEndpoint));
 
 	try {
-		if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-			printf("CFM: SS %s CF %s: got CFSQ %s [%s - %s) %lld - %lld, crp=%s\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       streamUID.toString().substr(0, 8).c_str(),
-			       req.range.begin.printable().c_str(),
-			       req.range.end.printable().c_str(),
-			       req.begin,
-			       req.end,
-			       req.canReadPopped ? "T" : "F");
+		if (DEBUG_CF_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedMutationsBegin", data->thisServerID)
+			    .detail("FeedID", req.rangeID)
+			    .detail("StreamUID", streamUID)
+			    .detail("Range", req.range)
+			    .detail("Begin", req.begin)
+			    .detail("End", req.end)
+			    .detail("CanReadPopped", req.canReadPopped);
 		}
 		data->activeFeedQueries++;
 
@@ -2434,14 +2338,6 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 		ASSERT(emptyInitialReply.minStreamVersion == invalidVersion);
 		req.reply.send(emptyInitialReply);
 
-		if (DEBUG_SS_CFM(data->thisServerID, req.rangeID, streamUID, req.begin)) {
-			printf("CFM: SS %s CF %s: CFSQ %s send empty initial version %lld\n",
-			       data->thisServerID.toString().substr(0, 4).c_str(),
-			       req.rangeID.printable().substr(0, 6).c_str(),
-			       streamUID.toString().substr(0, 8).c_str(),
-			       req.begin - 1);
-		}
-
 		loop {
 			Future<Void> onReady = req.reply.onReady();
 			if (atLatest && !onReady.isReady() && !removeUID) {
@@ -2462,14 +2358,6 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 			ChangeFeedStreamReply feedReply = _feedReply.first;
 			bool gotAll = _feedReply.second;
 
-			// TODO REMOVE debugging
-			if (feedReply.mutations.size() == 0) {
-				printf("CFM: SS %s CF %s: CFSQ %s empty results for begin=%lld\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       req.rangeID.printable().substr(0, 6).c_str(),
-				       streamUID.toString().substr(0, 8).c_str(),
-				       req.begin);
-			}
 			ASSERT(feedReply.mutations.size() > 0);
 			req.begin = feedReply.mutations.back().version + 1;
 			if (!atLatest && gotAll) {
@@ -2515,7 +2403,6 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 					when(wait(feed->second->newMutations.onTrigger())) {}
 					when(wait(req.end == std::numeric_limits<Version>::max() ? Future<Void>(Never())
 					                                                         : data->version.whenAtLeast(req.end))) {}
-					when(wait(delay(5.0))) {} // TODO REMOVE this once empty version logic is fully implemented
 				}
 				auto feed = data->uidChangeFeed.find(req.rangeID);
 				if (feed == data->uidChangeFeed.end() || feed->second->removing) {
@@ -2529,8 +2416,6 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 		}
 	} catch (Error& e) {
 		data->activeFeedQueries--;
-		// TODO REMOVE
-		printf("CFSQ %s got error %s\n", streamUID.toString().substr(0, 8).c_str(), e.name());
 		auto it = data->changeFeedClientVersions.find(req.reply.getEndpoint().getPrimaryAddress());
 		if (it != data->changeFeedClientVersions.end()) {
 			if (removeUID) {
@@ -4750,8 +4635,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
                                              KeyRange range,
                                              Version emptyVersion,
                                              Version beginVersion,
-                                             Version endVersion,
-                                             bool existing) {
+                                             Version endVersion) {
 
 	state Version startVersion = beginVersion;
 	startVersion = std::max(startVersion, emptyVersion + 1);
@@ -4760,17 +4644,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 
 	ASSERT(startVersion >= 0);
 
-	// TODO REMOVE
-	TraceEvent(SevDebug, "FetchChangeFeedStarting", data->thisServerID)
-	    .detail("RangeID", rangeId.printable())
-	    .detail("Range", range.toString())
-	    .detail("StartVersion", startVersion)
-	    .detail("EndVersion", endVersion)
-	    .detail("BeginVersion", beginVersion)
-	    .detail("EmptyVersion", emptyVersion)
-	    .detail("FetchVersion", changeFeedInfo->fetchVersion)
-	    .detail("DurableFetchVersion", changeFeedInfo->durableFetchVersion.get());
-
 	if (startVersion >= endVersion) {
 		TEST(true); // Change Feed popped before fetch
 		TraceEvent(SevDebug, "FetchChangeFeedNoOp", data->thisServerID)
@@ -4785,7 +4658,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	state Future<Void> feed = data->cx->getChangeFeedStream(
 	    feedResults, rangeId, startVersion, endVersion, range, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES, true);
 
-	// TODO remove debugging eventually?
 	state Version firstVersion = invalidVersion;
 	state Version lastVersion = invalidVersion;
 	state int64_t versionsFetched = 0;
@@ -4862,7 +4734,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 								for (auto& m : remoteResult[remoteLoc].mutations) {
 									DEBUG_MUTATION("ChangeFeedWriteMove", remoteVersion, m, data->thisServerID)
 									    .detail("Range", range)
-									    .detail("Existing", existing)
 									    .detail("ChangeFeedID", rangeId);
 								}
 							}
@@ -4885,7 +4756,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 							for (auto& m : remoteResult[remoteLoc].mutations) {
 								DEBUG_MUTATION("ChangeFeedWriteMoveIgnore", remoteVersion, m, data->thisServerID)
 								    .detail("Range", range)
-								    .detail("Existing", existing)
 								    .detail("ChangeFeedID", rangeId)
 								    .detail("EmptyVersion", changeFeedInfo->emptyVersion);
 							}
@@ -4963,7 +4833,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 		}
 	}
 
-	// TODO REMOVE?
 	TraceEvent(SevDebug, "FetchChangeFeedDone", data->thisServerID)
 	    .detail("RangeID", rangeId.printable())
 	    .detail("Range", range.toString())
@@ -4973,7 +4842,6 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	    .detail("FirstFetchedVersion", firstVersion)
 	    .detail("LastFetchedVersion", lastVersion)
 	    .detail("VersionsFetched", versionsFetched)
-	    .detail("Existing", existing)
 	    .detail("Removed", changeFeedInfo->removing);
 	return lastVersion;
 }
@@ -4997,10 +4865,6 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 
 	auto cleanupPending = data->changeFeedCleanupDurable.find(changeFeedInfo->id);
 	if (cleanupPending != data->changeFeedCleanupDurable.end()) {
-		/*printf("SS %s waiting for CF %s cleanup @ %lld\n",
-		       data->thisServerID.toString().substr(0, 4).c_str(),
-		       changeFeedInfo->id.toString().substr(0, 6).c_str(),
-		       cleanupPending->second);*/
 		TraceEvent(SevDebug, "FetchChangeFeedWaitCleanup", data->thisServerID)
 		    .detail("RangeID", changeFeedInfo->id.printable())
 		    .detail("Range", changeFeedInfo->range.toString())
@@ -5025,15 +4889,13 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 
 	loop {
 		try {
-			// TODO clean up existing param for !existing
 			Version maxFetched = wait(fetchChangeFeedApplier(data,
 			                                                 changeFeedInfo,
 			                                                 changeFeedInfo->id,
 			                                                 changeFeedInfo->range,
 			                                                 changeFeedInfo->emptyVersion,
 			                                                 beginVersion,
-			                                                 endVersion,
-			                                                 false));
+			                                                 endVersion));
 			data->fetchingChangeFeeds.insert(changeFeedInfo->id);
 			return maxFetched;
 		} catch (Error& e) {
@@ -5109,14 +4971,6 @@ ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data, KeyR
 				// reset fetch versions because everything previously fetched was cleaned up
 				changeFeedInfo->fetchVersion = invalidVersion;
 				changeFeedInfo->durableFetchVersion = NotifiedVersion();
-				// TODO REMOVE
-				TraceEvent(SevDebug, "FetchedChangeFeedInfoReset", data->thisServerID)
-				    .detail("RangeID", cfEntry.rangeId.printable())
-				    .detail("Range", cfEntry.range.toString())
-				    .detail("FetchVersion", fetchVersion)
-				    .detail("EmptyVersion", cfEntry.emptyVersion)
-				    .detail("CleanupVersion", feedCleanup->second)
-				    .detail("StopVersion", cfEntry.stopVersion);
 
 				// Since cleanup put a mutation in the log to delete the change feed data, put one in the log to restore
 				// it
@@ -5581,25 +5435,6 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		// Wait for the transferred version (and therefore the shard data) to be committed and durable.
 		wait(data->durableVersion.whenAtLeast(feedTransferredVersion));
 
-		// TODO if this works, remove all of the fetch version stuff
-		// Also wait on all fetched change feed data to become committed and durable
-		while (!feedFetchedVersions.empty()) {
-			auto feed = feedFetchedVersions.begin();
-			state Key feedId = feed->first;
-			Version maxFetched = feed->second;
-			feedFetchedVersions.erase(feed);
-
-			auto feedIt = data->uidChangeFeed.find(feedId);
-			/*if (feedIt != data->uidChangeFeed.end() && feedIt->second->durableFetchVersion.get() < maxFetched) {
-			    wait(feedIt->second->durableFetchVersion.whenAtLeast(maxFetched));
-			    // return to updateStorage
-			    wait(delay(0));
-			}*/
-			if (feedIt != data->uidChangeFeed.end()) {
-				ASSERT(feedIt->second->durableFetchVersion.get() >= maxFetched);
-			}
-		}
-
 		ASSERT(data->shards[shard->keys.begin]->assigned() &&
 		       data->shards[shard->keys.begin]->keys ==
 		           shard->keys); // We aren't changing whether the shard is assigned
@@ -5860,8 +5695,6 @@ void changeServerKeys(StorageServer* data,
 			}
 
 			if (!foundAssigned) {
-				// TODO REMOVE
-
 				Version durableVersion = data->data().getLatestVersion();
 				TraceEvent(SevDebug, "ChangeFeedCleanup", data->thisServerID)
 				    .detail("FeedID", f.first)
@@ -5880,19 +5713,6 @@ void changeServerKeys(StorageServer* data,
 				                                           changeFeedDurableKey(f.first, 0),
 				                                           changeFeedDurableKey(f.first, version)));
 
-				// do this cleanup later!!
-				/*auto rs = data->keyChangeFeed.modify(f.second);
-				for (auto r = rs.begin(); r != rs.end(); ++r) {
-				    auto& feedList = r->value();
-				    for (int i = 0; i < feedList.size(); i++) {
-				        if (feedList[i]->id == f.first) {
-				            swapAndPop(&feedList, i--);
-				        }
-				    }
-				}
-				data->keyChangeFeed.coalesce(f.second.contents());
-				*/
-
 				// We can't actually remove this change feed fully until the mutations clearing its data become durable.
 				// If the SS restarted at version R before the clearing mutations became durable at version D (R < D),
 				// then the restarted SS would restore the change feed clients would be able to read data and would miss
@@ -5904,7 +5724,6 @@ void changeServerKeys(StorageServer* data,
 					feed->second->removing = true;
 					feed->second->moved(feed->second->range);
 					feed->second->newMutations.trigger();
-					// data->uidChangeFeed.erase(feed);
 				}
 			} else {
 				// if just part of feed's range is moved away
@@ -6129,7 +5948,6 @@ private:
 			std::tie(changeFeedRange, popVersion, status) = decodeChangeFeedValue(m.param2);
 			auto feed = data->uidChangeFeed.find(changeFeedId);
 
-			// TODO REMOVE eventually
 			TraceEvent(SevDebug, "ChangeFeedPrivateMutation", data->thisServerID)
 			    .detail("RangeID", changeFeedId.printable())
 			    .detail("Range", changeFeedRange.toString())
@@ -6878,13 +6696,6 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 				}
 
 				if (info->second->fetchVersion != invalidVersion && !info->second->removing) {
-					// TODO REMOVE trace
-					TraceEvent(SevDebug, "UpdateStorageChangeFeedStart", data->thisServerID)
-					    .detail("RangeID", info->second->id.printable())
-					    .detail("Range", info->second->range.toString())
-					    .detail("FetchVersion", info->second->fetchVersion)
-					    .detail("StopVersion", info->second->stopVersion)
-					    .detail("Removing", info->second->removing);
 					feedFetchVersions.push_back(std::pair(info->second->id, info->second->fetchVersion));
 				}
 				// handle case where fetch had version ahead of last in-memory mutation
@@ -6958,15 +6769,6 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			// Don't update if the feed is pending cleanup. Either it will get cleaned up and destroyed, or it will get
 			// fetched again, where the fetch version will get reset.
 			if (info != data->uidChangeFeed.end() && !data->changeFeedCleanupDurable.count(info->second->id)) {
-				// TODO REMOVE trace
-				TraceEvent(SevDebug, "UpdateStorageChangeFeedDurable", data->thisServerID)
-				    .detail("RangeID", info->second->id.printable())
-				    .detail("Range", info->second->range.toString())
-				    .detail("FetchVersion", info->second->fetchVersion)
-				    .detail("OldDurableVersion", info->second->durableFetchVersion.get())
-				    .detail("NewDurableVersion", feedFetchVersions[curFeed].second)
-				    .detail("StopVersion", info->second->stopVersion)
-				    .detail("Removing", info->second->removing);
 				if (feedFetchVersions[curFeed].second > info->second->durableFetchVersion.get()) {
 					info->second->durableFetchVersion.set(feedFetchVersions[curFeed].second);
 				}
@@ -6986,11 +6788,6 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 				auto feed = data->uidChangeFeed.find(cfCleanup->first);
 				ASSERT(feed != data->uidChangeFeed.end());
 				if (feed->second->removing) {
-					// TODO  REMOVE
-					/*printf("DBG: SS %s Feed %s removing metadata @ %lld!\n",
-					       data->thisServerID.toString().substr(0, 4).c_str(),
-					       feed->first.printable().substr(0, 6).c_str(),
-					       cfCleanup->second);*/
 					auto rs = data->keyChangeFeed.modify(feed->second->range);
 					for (auto r = rs.begin(); r != rs.end(); ++r) {
 						auto& feedList = r->value();
@@ -7002,27 +6799,10 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 					}
 					data->keyChangeFeed.coalesce(feed->second->range.contents());
 
-					// TODO REMOVE
-					TraceEvent(SevDebug, "UpdateStorageChangeFeedCleanup", data->thisServerID)
-					    .detail("RangeID", feed->second->id.printable())
-					    .detail("Range", feed->second->range.toString());
-
 					data->uidChangeFeed.erase(feed);
 				} else {
 					TEST(true); // Feed re-fetched after remove
-					// TODO  REMOVE
-					/*printf("DBG: SS %s Feed %s not removing metadata @ %lld, must have been re-fetched after moved "
-					       "away!\n",
-					       data->thisServerID.toString().substr(0, 4).c_str(),
-					       feed->first.printable().substr(0, 6).c_str(),
-					       cfCleanup->second);*/
 				}
-
-				// TODO REMOVE
-				/*printf("DBG: SS %s Feed %s removing cleanup entry @ %lld!\n",
-				       data->thisServerID.toString().substr(0, 4).c_str(),
-				       feed->first.printable().substr(0, 6).c_str(),
-				       cfCleanup->second);*/
 				cfCleanup = data->changeFeedCleanupDurable.erase(cfCleanup);
 			} else {
 				cfCleanup++;

From 1f964ac0854e37b9f757030a14e7410f7ec2629a Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Wed, 9 Mar 2022 11:44:45 -0600
Subject: [PATCH 186/210] BM focused cleanup

---
 fdbclient/NativeAPI.actor.cpp                 |  52 +--
 fdbclient/ServerKnobs.cpp                     |   7 +-
 fdbclient/ServerKnobs.h                       |   5 +
 fdbserver/BlobManager.actor.cpp               | 302 ++++++++----------
 fdbserver/BlobWorker.actor.cpp                |  16 +-
 .../BlobGranuleCorrectnessWorkload.actor.cpp  |  31 +-
 6 files changed, 193 insertions(+), 220 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 1963fb908c..bb0d0143fc 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7010,6 +7010,32 @@ ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, LockAware
 	return Void();
 }
 
+ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
+                                                                                      bool primary,
+                                                                                      bool use_system_priority) {
+	state const Key readKey = perpetualStorageWiggleIDPrefix.withSuffix(primary ? "primary/"_sr : "remote/"_sr);
+	state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(readKey,
+	                                                                                          IncludeVersion());
+	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	state std::vector<std::pair<UID, StorageWiggleValue>> res;
+	// read the wiggling pairs
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			if (use_system_priority) {
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			}
+			wait(store(res, metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY)));
+			wait(tr->commit());
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+	return res;
+}
+
 ACTOR Future<Void> splitStorageMetricsStream(PromiseStream<Key> resultStream,
                                              Database cx,
                                              KeyRange keys,
@@ -7111,32 +7137,6 @@ Future<Void> DatabaseContext::splitStorageMetricsStream(const PromiseStream<Key>
 	    resultStream, Database(Reference<DatabaseContext>::addRef(this)), keys, limit, estimated);
 }
 
-ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
-                                                                                      bool primary,
-                                                                                      bool use_system_priority) {
-	state const Key readKey = perpetualStorageWiggleIDPrefix.withSuffix(primary ? "primary/"_sr : "remote/"_sr);
-	state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(readKey,
-	                                                                                          IncludeVersion());
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state std::vector<std::pair<UID, StorageWiggleValue>> res;
-	// read the wiggling pairs
-	loop {
-		try {
-			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-			if (use_system_priority) {
-				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			}
-			wait(store(res, metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY)));
-			wait(tr->commit());
-			break;
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-	return res;
-}
-
 ACTOR Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(Database cx,
                                                                 KeyRange keys,
                                                                 StorageMetrics limit,
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 58d678a310..e8ddd6b7d9 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -821,7 +821,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
     init( ENABLE_ENCRYPT_KEY_PROXY,                            false );
 
 	// Blob granlues
-	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space, eventually
+	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually
 	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards || (randomize && BUGGIFY) ) { deterministicRandom()->random01() < 0.1 ? BG_SNAPSHOT_FILE_TARGET_BYTES /= 100 : BG_SNAPSHOT_FILE_TARGET_BYTES /= 10; }
 	init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 );
 	init( BG_DELTA_FILE_TARGET_BYTES,   BG_DELTA_BYTES_BEFORE_COMPACT/10 );
@@ -829,6 +829,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BLOB_WORKER_TIMEOUT,                                  10.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0;
 	init( BLOB_WORKER_REQUEST_TIMEOUT,                           5.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0;
 	init( BLOB_WORKERLIST_FETCH_INTERVAL,                        1.0 );
+	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = deterministicRandom()->randomInt(5, 15);
+
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN,                        0.1 );
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX,                        5.0 );
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,                   1.5 );
 
 	// clang-format on
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 4a783da91d..0a0023719c 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -780,6 +780,11 @@ public:
 	double BLOB_WORKER_TIMEOUT; // Blob Manager's reaction time to a blob worker failure
 	double BLOB_WORKER_REQUEST_TIMEOUT; // Blob Worker's server-side request timeout
 	double BLOB_WORKERLIST_FETCH_INTERVAL;
+	int BG_MAX_SPLIT_FANOUT; // Decreasing this knob can be unsafe
+
+	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
+	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX;
+	double BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT;
 
 	ServerKnobs(Randomize, ClientKnobs*, IsSimulated);
 	void initialize(Randomize, ClientKnobs*, IsSimulated);
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 7c16df3c11..4f94391dd2 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,9 +43,12 @@
 #include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+/*
+ * The Blob Manager is responsible for managing range granules, and recruiting and monitoring Blob Workers.
+ */
+
 #define BM_DEBUG false
 
-// TODO add comments + documentation
 void handleClientBlobRange(KeyRangeMap<bool>* knownBlobRanges,
                            Arena& ar,
                            VectorRef<KeyRangeRef>* rangesToAdd,
@@ -103,7 +106,7 @@ void updateClientBlobRanges(KeyRangeMap<bool>* knownBlobRanges,
 	// worker. for any range that isn't set in results that is set in ranges, revoke the range from the
 	// worker. and, update ranges to match results as you go
 
-	// FIXME: could change this to O(N) instead of O(NLogN) by doing a sorted merge instead of requesting the
+	// SOMEDAY: could change this to O(N) instead of O(NLogN) by doing a sorted merge instead of requesting the
 	// intersection for each insert, but this operation is pretty infrequent so it's probably not necessary
 	if (dbBlobRanges.size() == 0) {
 		// special case. Nothing in the DB, reset knownBlobRanges and revoke all existing ranges from workers
@@ -191,7 +194,7 @@ struct RangeAssignment {
 	Optional<RangeRevokeData> revoke;
 };
 
-// TODO: track worker's reads/writes eventually
+// SOMEDAY: track worker's reads/writes eventually
 struct BlobWorkerStats {
 	int numGranulesAssigned;
 
@@ -246,87 +249,82 @@ struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 	BlobManagerData(UID id, Database db, Optional<Key> dcId)
 	  : id(id), db(db), dcId(dcId), knownBlobRanges(false, normalKeys.end),
 	    restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), recruitingStream(0) {}
-
-	// TODO REMOVE
-	~BlobManagerData() {
-		if (BM_DEBUG) {
-			fmt::print("Destroying blob manager data for {0} {1}\n", epoch, id.toString());
-		}
-	}
 };
 
-ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<ReadYourWritesTransaction> tr,
+ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData> bmData,
                                                        KeyRange range,
                                                        bool writeHot) {
-	// TODO is it better to just pass empty metrics to estimated?
-	// redo split if previous txn failed to calculate it
-	loop {
-		try {
-			if (BM_DEBUG) {
-				fmt::print("Splitting new range [{0} - {1}): {2}\n",
-				           range.begin.printable(),
-				           range.end.printable(),
-				           writeHot ? "hot" : "normal");
-			}
-			state StorageMetrics estimated =
-			    wait(tr->getTransaction().getDatabase()->getStorageMetrics(range, CLIENT_KNOBS->TOO_MANY));
-
-			if (BM_DEBUG) {
-				fmt::print("Estimated bytes for [{0} - {1}): {2}\n",
-				           range.begin.printable(),
-				           range.end.printable(),
-				           estimated.bytes);
-			}
-
-			if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES || writeHot) {
-				// only split on bytes and write rate
-				state StorageMetrics splitMetrics;
-				splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES;
-				splitMetrics.bytesPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
-				if (writeHot) {
-					splitMetrics.bytesPerKSecond =
-					    std::min(splitMetrics.bytesPerKSecond, estimated.bytesPerKSecond / 2);
-					splitMetrics.bytesPerKSecond =
-					    std::max(splitMetrics.bytesPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
-				}
-				splitMetrics.iosPerKSecond = splitMetrics.infinity;
-				splitMetrics.bytesReadPerKSecond = splitMetrics.infinity;
-
-				state PromiseStream<Key> resultStream;
-				state Standalone<VectorRef<KeyRef>> keys;
-				state Future<Void> streamFuture = tr->getTransaction().getDatabase()->splitStorageMetricsStream(
-				    resultStream, range, splitMetrics, estimated);
-				loop {
-					try {
-						Key k = waitNext(resultStream.getFuture());
-						keys.push_back_deep(keys.arena(), k);
-					} catch (Error& e) {
-						if (e.code() != error_code_end_of_stream) {
-							throw;
-						}
-						break;
-					}
-				}
-
-				ASSERT(keys.size() >= 2);
-				ASSERT(keys.front() == range.begin);
-				ASSERT(keys.back() == range.end);
-				return keys;
-			} else {
-				if (BM_DEBUG) {
-					printf("Not splitting range\n");
-				}
-				Standalone<VectorRef<KeyRef>> keys;
-				keys.push_back_deep(keys.arena(), range.begin);
-				keys.push_back_deep(keys.arena(), range.end);
-				return keys;
-			}
-		} catch (Error& e) {
-			if (BM_DEBUG) {
-				printf("Splitting range got error %s\n", e.name());
-			}
-			wait(tr->onError(e));
+	try {
+		if (BM_DEBUG) {
+			fmt::print("Splitting new range [{0} - {1}): {2}\n",
+			           range.begin.printable(),
+			           range.end.printable(),
+			           writeHot ? "hot" : "normal");
 		}
+		state StorageMetrics estimated = wait(bmData->db->getStorageMetrics(range, CLIENT_KNOBS->TOO_MANY));
+
+		if (BM_DEBUG) {
+			fmt::print("Estimated bytes for [{0} - {1}): {2}\n",
+			           range.begin.printable(),
+			           range.end.printable(),
+			           estimated.bytes);
+		}
+
+		if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES || writeHot) {
+			// only split on bytes and write rate
+			state StorageMetrics splitMetrics;
+			splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES;
+			splitMetrics.bytesPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
+			if (writeHot) {
+				splitMetrics.bytesPerKSecond = std::min(splitMetrics.bytesPerKSecond, estimated.bytesPerKSecond / 2);
+				splitMetrics.bytesPerKSecond =
+				    std::max(splitMetrics.bytesPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
+			}
+			splitMetrics.iosPerKSecond = splitMetrics.infinity;
+			splitMetrics.bytesReadPerKSecond = splitMetrics.infinity;
+
+			state PromiseStream<Key> resultStream;
+			state Standalone<VectorRef<KeyRef>> keys;
+			state Future<Void> streamFuture =
+			    bmData->db->splitStorageMetricsStream(resultStream, range, splitMetrics, estimated);
+			loop {
+				try {
+					Key k = waitNext(resultStream.getFuture());
+					keys.push_back_deep(keys.arena(), k);
+				} catch (Error& e) {
+					if (e.code() != error_code_end_of_stream) {
+						throw;
+					}
+					break;
+				}
+			}
+
+			ASSERT(keys.size() >= 2);
+			ASSERT(keys.front() == range.begin);
+			ASSERT(keys.back() == range.end);
+			return keys;
+		} else {
+			if (BM_DEBUG) {
+				printf("Not splitting range\n");
+			}
+			Standalone<VectorRef<KeyRef>> keys;
+			keys.push_back_deep(keys.arena(), range.begin);
+			keys.push_back_deep(keys.arena(), range.end);
+			return keys;
+		}
+	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled) {
+			throw e;
+		}
+		// SplitStorageMetrics explicitly has a SevError if it gets an error, so no errors should propagate here
+		TraceEvent(SevError, "BlobManagerUnexpectedErrorSplitRange", bmData->id)
+		    .error(e)
+		    .detail("Epoch", bmData->epoch);
+		ASSERT_WE_THINK(false);
+
+		// if not simulation, kill the BM
+		bmData->iAmReplaced.sendError(e);
+		throw e;
 	}
 }
 
@@ -335,7 +333,6 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<ReadYourWritesT
 ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
 	// wait until there are BWs to pick from
 	while (bmData->workerStats.size() == 0) {
-		// TODO REMOVE
 		if (BM_DEBUG) {
 			fmt::print("BM {0} waiting for blob workers before assigning granules\n", bmData->epoch);
 		}
@@ -450,8 +447,17 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			return Void();
 		}
 
-		// TODO confirm: using reliable delivery this should only trigger if the worker is marked as failed, right?
-		// So assignment needs to be retried elsewhere, and a revoke is trivially complete
+		if (e.code() != error_code_broken_promise && e.code() != error_code_no_more_servers) {
+			TraceEvent(SevWarn, "BlobManagerUnexpectedErrorDoRangeAssignment", bmData->id)
+			    .error(e)
+			    .detail("Epoch", bmData->epoch);
+			ASSERT_WE_THINK(false);
+			bmData->iAmReplaced.sendError(e);
+			throw;
+		}
+
+		// We use reliable delivery (getReply), so the broken_promise means the worker is dead, and we may need to retry
+		// somewhere else
 		if (assignment.isAssign) {
 			if (BM_DEBUG) {
 				fmt::print("BM got error {0} assigning range [{1} - {2}) to worker {3}, requeueing\n",
@@ -567,8 +573,6 @@ ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 				bmData->workerStats[workerId].numGranulesAssigned += 1;
 			}
 
-			// FIXME: if range is assign, have some sort of semaphore for outstanding assignments so we don't assign
-			// a ton ranges at once and blow up FDB with reading initial snapshots.
 			bmData->assignsInProgress.insert(assignment.keyRange,
 			                                 doRangeAssignment(bmData, assignment, workerId, seqNo));
 		} else {
@@ -647,13 +651,6 @@ ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData,
 				tr->setOption(FDBTransactionOptions::Option::ACCESS_SYSTEM_KEYS);
 				wait(checkManagerLock(tr, bmData));
 				while (i + j < boundaries.size() - 1 && j < transactionChunkSize) {
-					// TODO REMOVE
-					if (BM_DEBUG) {
-						fmt::print("BM {0} Persisting initial mapping for [{1} - {2})\n",
-						           bmData->epoch,
-						           boundaries[i + j].printable(),
-						           boundaries[i + j + 1].printable());
-					}
 					// set to empty UID - no worker assigned yet
 					wait(krmSetRange(tr,
 					                 blobGranuleMappingKeys.begin,
@@ -662,19 +659,8 @@ ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData,
 					j++;
 				}
 				wait(tr->commit());
-				if (BM_DEBUG) {
-					for (int k = 0; k < j; k++) {
-						fmt::print("BM {0} Persisted initial mapping for [{1} - {2})\n",
-						           bmData->epoch,
-						           boundaries[i + k].printable(),
-						           boundaries[i + k + 1].printable());
-					}
-				}
 				break;
 			} catch (Error& e) {
-				if (BM_DEBUG) {
-					fmt::print("BM {} Persisting initial mapping got error {}\n", bmData->epoch, e.name());
-				}
 				wait(tr->onError(e));
 				j = 0;
 			}
@@ -684,8 +670,6 @@ ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData,
 	return Void();
 }
 
-// FIXME: this does all logic in one transaction. Adding a giant range to an existing database to blobify would
-// require doing a ton of storage metrics calls, which we should split up across multiple transactions likely.
 ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 	state Optional<Value> lastChangeKeyValue;
 	state bool needToCoalesce = bmData->epoch > 1;
@@ -703,10 +687,20 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 				// read change key at this point along with ranges
 				state Optional<Value> ckvBegin = wait(tr->get(blobRangeChangeKey));
 
-				// TODO probably knobs here? This should always be pretty small though
-				RangeResult results = wait(krmGetRanges(
-				    tr, blobRangeKeys.begin, KeyRange(normalKeys), 10000, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
-				ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+				state RangeResult results = wait(krmGetRanges(tr,
+				                                              blobRangeKeys.begin,
+				                                              KeyRange(normalKeys),
+				                                              CLIENT_KNOBS->TOO_MANY,
+				                                              GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+				ASSERT_WE_THINK(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+				if (results.more || results.size() >= CLIENT_KNOBS->TOO_MANY) {
+					TraceEvent(SevError, "BlobManagerTooManyClientRanges", bmData->id)
+					    .detail("Epoch", bmData->epoch)
+					    .detail("ClientRanges", results.size() - 1);
+					wait(delay(600));
+					bmData->iAmReplaced.sendError(internal_error());
+					throw internal_error();
+				}
 
 				state Arena ar;
 				ar.dependsOn(results.arena());
@@ -743,7 +737,7 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 				state std::vector<Future<Standalone<VectorRef<KeyRef>>>> splitFutures;
 				// Divide new ranges up into equal chunks by using SS byte sample
 				for (KeyRangeRef range : rangesToAdd) {
-					splitFutures.push_back(splitRange(tr, range, false));
+					splitFutures.push_back(splitRange(bmData, range, false));
 				}
 
 				for (auto f : splitFutures) {
@@ -847,7 +841,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 	state int64_t newLockSeqno = -1;
 
 	// first get ranges to split
-	Standalone<VectorRef<KeyRef>> _newRanges = wait(splitRange(tr, granuleRange, writeHot));
+	Standalone<VectorRef<KeyRef>> _newRanges = wait(splitRange(bmData, granuleRange, writeHot));
 	newRanges = _newRanges;
 
 	ASSERT(newRanges.size() >= 2);
@@ -868,10 +862,9 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		return Void();
 	}
 
-	// TODO KNOB for this.
-	// Enforce max split fanout of 10 for performance reasons
-	int maxSplitFanout = 10;
-	if (newRanges.size() >= maxSplitFanout + 2) { // +2 because this is boundaries, so N keys would have N+1 bounaries.
+	// Enforce max split fanout for performance reasons. This mainly happens when a blob worker is behind.
+	if (newRanges.size() >=
+	    SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2) { // +2 because this is boundaries, so N keys would have N+1 bounaries.
 		TEST(true); // downsampling granule split because fanout too high
 		Standalone<VectorRef<KeyRef>> coalescedRanges;
 		coalescedRanges.arena().dependsOn(newRanges.arena());
@@ -879,15 +872,17 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 
 		// since we include start + end boundaries here, only need maxSplitFanout-1 split boundaries to produce
 		// maxSplitFanout granules
-		downsampleSplit(newRanges, coalescedRanges, 1, newRanges.size() - 1, maxSplitFanout - 1);
+		downsampleSplit(newRanges, coalescedRanges, 1, newRanges.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT - 1);
 
 		coalescedRanges.push_back(coalescedRanges.arena(), newRanges.back());
-		ASSERT(coalescedRanges.size() == maxSplitFanout + 1);
+		ASSERT(coalescedRanges.size() == SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 		if (BM_DEBUG) {
-			fmt::print("Downsampled split from {0} -> {1} granules", newRanges.size() - 1, maxSplitFanout);
+			fmt::print(
+			    "Downsampled split from {0} -> {1} granules", newRanges.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT);
 		}
 
 		newRanges = coalescedRanges;
+		ASSERT(newRanges.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 	}
 
 	if (BM_DEBUG) {
@@ -971,20 +966,9 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			// acquire granule lock so nobody else can make changes to this granule.
 			tr->set(lockKey, blobGranuleLockValueFor(bmData->epoch, newLockSeqno, std::get<2>(prevGranuleLock)));
 
-			// set up split metadata
-			/*fmt::print("Persisting granule split {0} [{1} - {2})\n",
-			           granuleID.toString().substr(0, 6),
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable());*/
-
 			// set up splits in granule mapping, but point each part to the old owner (until they get reassigned)
 			state int i;
 			for (i = 0; i < newRanges.size() - 1; i++) {
-				/*fmt::print("    {0} [{1} - {2})\n",
-				           newGranuleIDs[i].toString().substr(0, 6),
-				           newRanges[i].printable(),
-				           newRanges[i + 1].printable());*/
-
 				Key splitKey = blobGranuleSplitKeyFor(granuleID, newGranuleIDs[i]);
 
 				tr->atomicOp(splitKey,
@@ -998,11 +982,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				historyValue.parentGranules.push_back(historyValue.arena(),
 				                                      std::pair(granuleRange, granuleStartVersion));
 
-				/*fmt::print("Creating history entry [{0} - {1}) - [{2} - {3})\n",
-				           newRanges[i].printable(),
-				           newRanges[i + 1].printable(),
-				           granuleStartVersion,
-				           latestVersion);*/
 				tr->set(historyKey, blobGranuleHistoryValueFor(historyValue));
 
 				// split the assignment but still pointing to the same worker
@@ -1090,11 +1069,6 @@ ACTOR Future<Void> haltBlobWorker(Reference<BlobManagerData> bmData, BlobWorkerI
 			if (e.code() == error_code_operation_cancelled) {
 				throw;
 			}
-			// TODO REMOVE
-			fmt::print("BM {0} got error {1} trying to halt blob worker {2}\n",
-			           bmData->epoch,
-			           e.name(),
-			           bwInterf.id().toString());
 			if (e.code() != error_code_blob_manager_replaced) {
 				break;
 			}
@@ -1185,8 +1159,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 	// wait for blob manager to be done recovering, so it has initial granule mapping and worker data
 	wait(bmData->doneRecovering.getFuture());
 
-	// TODO knob?
-	state double backoff = 0.1;
+	state double backoff = SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
 	loop {
 		try {
 			state ReplyPromiseStream<GranuleStatusReply> statusStream =
@@ -1207,7 +1180,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 					           rep.writeHotSplit ? "hot" : "normal");
 				}
 				// if we get a reply from the stream, reset backoff
-				backoff = 0.1;
+				backoff = SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
 				if (rep.epoch > bmData->epoch) {
 					if (BM_DEBUG) {
 						fmt::print("BM heard from BW {0} that there is a new manager with higher epoch\n",
@@ -1218,7 +1191,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 					}
 				}
 
-				// TODO maybe this won't be true eventually, but right now the only time the blob worker reports
+				// This won't be true eventually, but right now the only time the blob worker reports
 				// back is to split the range.
 				ASSERT(rep.doSplit);
 
@@ -1316,17 +1289,16 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 			ASSERT(e.code() != error_code_end_of_stream);
 			if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_connection_failed) {
 				wait(delay(backoff));
-				backoff = std::min(backoff * 1.5, 5.0);
+				backoff = std::min(backoff * SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,
+				                   SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX);
 				continue;
 			} else {
-				if (BM_DEBUG) {
-					fmt::print(
-					    "BM got unexpected error {0} monitoring BW {1} status\n", e.name(), bwInterf.id().toString());
-				}
-				// TODO change back from SevError?
-				TraceEvent(SevError, "BWStatusMonitoringFailed", bmData->id)
+				TraceEvent(SevError, "BlobManagerUnexpectedErrorStatusMonitoring", bmData->id)
 				    .error(e)
-				    .detail("BlobWorkerID", bwInterf.id());
+				    .detail("Epoch", bmData->epoch);
+				ASSERT_WE_THINK(false);
+				// if not simulation, kill the BM
+				bmData->iAmReplaced.sendError(e);
 				throw e;
 			}
 		}
@@ -1361,14 +1333,17 @@ ACTOR Future<Void> monitorBlobWorker(Reference<BlobManagerData> bmData, BlobWork
 			    "BM {0} got monitoring error {1} from BW {2}\n", bmData->epoch, e.name(), bwInterf.id().toString());
 		}
 
-		// TODO: re-evaluate the expected errors here once wait failure issue is resolved
 		// Expected errors here are: [broken_promise]
 		if (e.code() != error_code_broken_promise) {
 			if (BM_DEBUG) {
 				fmt::print("BM got unexpected error {0} monitoring BW {1}\n", e.name(), bwInterf.id().toString());
 			}
-			// TODO change back from SevError?
-			TraceEvent(SevError, "BWMonitoringFailed", bmData->id).error(e).detail("BlobWorkerID", bwInterf.id());
+			TraceEvent(SevError, "BlobManagerUnexpectedErrorMonitorBW", bmData->id)
+			    .error(e)
+			    .detail("Epoch", bmData->epoch);
+			ASSERT_WE_THINK(false);
+			// if not simulation, kill the BM
+			bmData->iAmReplaced.sendError(e);
 			throw e;
 		}
 	}
@@ -1536,7 +1511,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 	workerAssignments.insert(normalKeys, std::tuple(UID(), 0, 0));
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 
-	// TODO KNOB
+	// FIXME: use range stream instead
 	state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : 10000;
 
 	if (BM_DEBUG) {
@@ -1559,13 +1534,11 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		aliveAssignments.push_back(timeout(brokenPromiseToNever(it.granuleAssignmentsRequest.getReply(req)),
 		                                   SERVER_KNOBS->BLOB_WORKER_TIMEOUT));
 	}
-	waitForAll(aliveAssignments);
 
 	state std::vector<std::pair<UID, KeyRange>> outOfDateAssignments;
 	state int successful = 0;
 	state int assignIdx = 0;
 
-	// FIXME: more CPU efficient to do sorted merge of assignments?
 	for (; assignIdx < aliveAssignments.size(); assignIdx++) {
 		Optional<GetGranuleAssignmentsReply> reply = wait(aliveAssignments[assignIdx]);
 		UID workerId = startingWorkers[assignIdx].id();
@@ -1595,7 +1568,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 				bmData->workerStats[workerId].numGranulesAssigned = reply.get().assignments.size();
 			}
 		} else {
-			// TODO mark as failed and kill it
+			// SOMEDAY: mark as failed and kill it
 			if (BM_DEBUG) {
 				fmt::print("  Worker {}: failed\n", workerId.toString().substr(0, 5));
 			}
@@ -1621,7 +1594,6 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-			// TODO: replace row limit with knob
 			KeyRange nextRange(KeyRangeRef(beginKey, blobGranuleMappingKeys.end));
 			// using the krm functions can produce incorrect behavior here as it does weird stuff with beginKey
 			state GetRangeLimits limits(rowLimit, GetRangeLimits::BYTE_LIMIT_UNLIMITED);
@@ -1630,9 +1602,6 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 
 			// Add the mappings to our in memory key range map
 			for (int rangeIdx = 0; rangeIdx < results.size() - 1; rangeIdx++) {
-				// TODO REMOVE asserts eventually
-				ASSERT(results[rangeIdx].key.startsWith(blobGranuleMappingKeys.begin));
-				ASSERT(results[rangeIdx + 1].key.startsWith(blobGranuleMappingKeys.begin));
 				Key granuleStartKey = results[rangeIdx].key.removePrefix(blobGranuleMappingKeys.begin);
 				Key granuleEndKey = results[rangeIdx + 1].key.removePrefix(blobGranuleMappingKeys.begin);
 				if (results[rangeIdx].value.size()) {
@@ -1726,9 +1695,6 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		int64_t epoch = std::get<1>(range.value());
 		int64_t seqno = std::get<2>(range.value());
 		if (epoch == 0 && seqno == 0) {
-			/*if (BM_DEBUG) {
-			    fmt::print("  [{0} - {1}) invalid\n", range.begin().printable(), range.end().printable());
-			}*/
 			continue;
 		}
 
@@ -1983,7 +1949,6 @@ ACTOR Future<Void> haltBlobGranules(Reference<BlobManagerData> bmData) {
 	std::vector<BlobWorkerInterface> blobWorkers = wait(getBlobWorkers(bmData->db));
 	std::vector<Future<Void>> deregisterBlobWorkers;
 	for (auto& worker : blobWorkers) {
-		// TODO: send a special req to blob workers so they clean up granules/CFs
 		bmData->addActor.send(haltBlobWorker(bmData, worker));
 		deregisterBlobWorkers.emplace_back(deregisterBlobWorker(bmData, worker));
 	}
@@ -2010,7 +1975,7 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobManagerData> bmData, U
 /*
  * Deletes all files pertaining to the granule with id granuleId and
  * also removes the history entry for this granule from the system keyspace
- * TODO ensure cannot fully delete granule that is still splitting!
+ * TODO: ensure cannot fully delete granule that is still splitting!
  */
 ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self, UID granuleId, KeyRef historyKey) {
 	if (BM_DEBUG) {
@@ -2194,7 +2159,6 @@ ACTOR Future<Void> pruneRange(Reference<BlobManagerData> self, KeyRangeRef range
 	}
 
 	// queue of <range, startVersion, endVersion> for BFS traversal of history
-	// TODO: consider using GranuleHistoryEntry, but that also makes it a little messy
 	state std::queue<std::tuple<KeyRange, Version, Version>> historyEntryQueue;
 
 	// stacks of <granuleId, historyKey> and <granuleId> to track which granules to delete
@@ -2569,7 +2533,6 @@ ACTOR Future<Void> blobManagerExclusionSafetyCheck(Reference<BlobManagerData> se
 		TraceEvent("BMExclusionSafetyCheckNoWorkers", self->id).log();
 		reply.safe = false;
 	} else {
-		// TODO REMOVE prints
 		std::set<UID> remainingWorkers;
 		for (auto& worker : self->workersById) {
 			remainingWorkers.insert(worker.first);
@@ -2627,7 +2590,6 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 		self->addActor.send(chaosRangeMover(self));
 	}
 
-	// TODO probably other things here eventually
 	try {
 		loop choose {
 			when(wait(self->iAmReplaced.getFuture())) {
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index f8afcf0e73..2197885b6b 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -59,7 +59,7 @@ struct GranuleStartState {
 	Optional<GranuleHistory> history;
 };
 
-// TODO add global byte limit for pending and buffered deltas
+// FIXME: add global byte limit for pending and buffered deltas
 struct GranuleMetadata : NonCopyable, ReferenceCounted<GranuleMetadata> {
 	KeyRange keyRange;
 
@@ -103,7 +103,6 @@ struct GranuleMetadata : NonCopyable, ReferenceCounted<GranuleMetadata> {
 	}
 };
 
-// TODO: rename this struct
 struct GranuleRangeMetadata {
 	int64_t lastEpoch;
 	int64_t lastSeqno;
@@ -149,6 +148,7 @@ struct GranuleHistoryEntry : NonCopyable, ReferenceCounted<GranuleHistoryEntry>
 	  : range(range), granuleID(granuleID), startVersion(startVersion), endVersion(endVersion) {}
 };
 
+// TODO: add limit on active fdb initial snapshots we can do at once per worker (knob + flow lock)
 struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 	UID id;
 	Database db;
@@ -320,10 +320,16 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
                                            BlobGranuleSplitState newState) {
 	state KeyRange currentRange = blobGranuleSplitKeyRangeFor(parentGranuleID);
 
-	RangeResult totalState = wait(tr->getRange(currentRange, 100));
+	state RangeResult totalState = wait(tr->getRange(currentRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2));
 	// TODO is this explicit conflit range necessary with the above read?
 	tr->addWriteConflictRange(currentRange);
-	ASSERT(!totalState.more);
+	ASSERT_WE_THINK(!totalState.more && totalState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
+	// maybe someone decreased the knob, we should gracefully handle it not in simulation
+	if (totalState.more || totalState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1) {
+		RangeResult tryAgain = wait(tr->getRange(currentRange, 10000));
+		ASSERT(!tryAgain.more);
+		totalState = tryAgain;
+	}
 
 	if (totalState.empty()) {
 		ASSERT(newState == BlobGranuleSplitState::Done);
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index 352511fbb8..e6ec6013c7 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -191,7 +191,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 				// set up directory with its own randomness
 				uint32_t dirId = i * clientCount + clientId;
 				if (BGW_DEBUG) {
-					printf("Client %d/%d creating directory %d\n", clientId, clientCount, dirId);
+					fmt::print("Client {0}/{1} creating directory {2}\n", clientId, clientCount, dirId);
 				}
 				directories.push_back(makeReference<ThreadData>(dirId, targetByteRate));
 				targetByteRate /= skewMultiplier;
@@ -306,8 +306,6 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 				Version readVersion = wait(tr->getReadVersion());
 				return readVersion;
 			} catch (Error& e) {
-				// TODO REMOVE print
-				printf("BGV GRV got error %s\n", e.name());
 				wait(tr->onError(e));
 			}
 		}
@@ -325,10 +323,10 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 				state Version readVersion = rv;
 				std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blob =
 				    wait(self->readFromBlob(cx, self, threadData->directoryRange, readVersion));
-				printf("Directory %d got %s RV %lld\n",
-				       threadData->directoryID,
-				       doSetup ? "initial" : "final",
-				       readVersion);
+				fmt::print("Directory {0} got {1} RV {2}\n",
+				           threadData->directoryID,
+				           doSetup ? "initial" : "final",
+				           readVersion);
 				threadData->minSuccessfulReadVersion = readVersion;
 				return Void();
 			} catch (Error& e) {
@@ -370,7 +368,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		           format("%08x", endKey),
 		           readVersion);
 		if (lastMatching.present()) {
-			printf("    last correct: %s\n", lastMatching.get().printable().c_str());
+			fmt::print("    last correct: {}\n", lastMatching.get().printable());
 		}
 		if (expectedValue.present() || blobValue.present()) {
 			// value mismatch
@@ -386,17 +384,17 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 			fmt::print("      Actual Key: {0}\n", blobKey.present() ? blobKey.get().printable() : "<missing>");
 		}
 
-		printf("Chunks:\n");
+		fmt::print("Chunks:\n");
 		for (auto& chunk : blob.second) {
-			printf("[%s - %s)\n", chunk.keyRange.begin.printable().c_str(), chunk.keyRange.end.printable().c_str());
+			fmt::print("[{0} - {1})\n", chunk.keyRange.begin.printable(), chunk.keyRange.end.printable());
 
-			printf("  SnapshotFile:\n    %s\n",
-			       chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : "<none>");
-			printf("  DeltaFiles:\n");
+			fmt::print("  SnapshotFile:\n    {}\n",
+			           chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : "<none>");
+			fmt::print("  DeltaFiles:\n");
 			for (auto& df : chunk.deltaFiles) {
-				printf("    %s\n", df.toString().c_str());
+				fmt::print("    {}\n", df.toString());
 			}
-			printf("  Deltas: (%d)", chunk.newDeltas.size());
+			fmt::print("  Deltas: ({})", chunk.newDeltas.size());
 			if (chunk.newDeltas.size() > 0) {
 				fmt::print(" with version [{0} - {1}]",
 				           chunk.newDeltas[0].version,
@@ -410,9 +408,6 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 	Value genVal(uint32_t val, uint16_t valLen) {
 		std::string v(valLen, 'x');
 		auto valFormatted = format("%08x", val);
-		if (valFormatted.size() > v.size()) {
-			printf("valFormatted=%d, v.size=%d\n", valFormatted.size(), v.size());
-		}
 		ASSERT(valFormatted.size() <= v.size());
 
 		for (int i = 0; i < valFormatted.size(); i++) {

From c8c97e0256bf3a78d5c3554cd475ebdb9921ee03 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 09:22:53 -0600
Subject: [PATCH 187/210] Blob Worker focused cleanup

---
 fdbclient/ServerKnobs.cpp       |  12 +-
 fdbclient/ServerKnobs.h         |   5 +-
 fdbserver/BlobManager.actor.cpp |  21 +-
 fdbserver/BlobWorker.actor.cpp  | 565 ++++++++++++--------------------
 4 files changed, 231 insertions(+), 372 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index e8ddd6b7d9..bc6f181c79 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -825,15 +825,19 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards || (randomize && BUGGIFY) ) { deterministicRandom()->random01() < 0.1 ? BG_SNAPSHOT_FILE_TARGET_BYTES /= 100 : BG_SNAPSHOT_FILE_TARGET_BYTES /= 10; }
 	init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 );
 	init( BG_DELTA_FILE_TARGET_BYTES,   BG_DELTA_BYTES_BEFORE_COMPACT/10 );
+	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15);
+	init( BG_HOT_SNAPSHOT_VERSIONS,                          5000000 );
 
+	init( BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM,                8 ); if( randomize && BUGGIFY ) BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM = 1;
 	init( BLOB_WORKER_TIMEOUT,                                  10.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0;
 	init( BLOB_WORKER_REQUEST_TIMEOUT,                           5.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0;
 	init( BLOB_WORKERLIST_FETCH_INTERVAL,                        1.0 );
-	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = deterministicRandom()->randomInt(5, 15);
+	init( BLOB_WORKER_BATCH_GRV_INTERVAL,                        0.1 );
+	
 
-	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN,                        0.1 );
-	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX,                        5.0 );
-	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,                   1.5 );
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN,                   0.1 );
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX,                   5.0 );
+	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,              1.5 );
 
 	// clang-format on
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 0a0023719c..a707580ae8 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -776,11 +776,14 @@ public:
 	int BG_SNAPSHOT_FILE_TARGET_BYTES;
 	int BG_DELTA_FILE_TARGET_BYTES;
 	int BG_DELTA_BYTES_BEFORE_COMPACT;
+	int BG_MAX_SPLIT_FANOUT;
+	int BG_HOT_SNAPSHOT_VERSIONS;
 
+	int BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM;
 	double BLOB_WORKER_TIMEOUT; // Blob Manager's reaction time to a blob worker failure
 	double BLOB_WORKER_REQUEST_TIMEOUT; // Blob Worker's server-side request timeout
 	double BLOB_WORKERLIST_FETCH_INTERVAL;
-	int BG_MAX_SPLIT_FANOUT; // Decreasing this knob can be unsafe
+	double BLOB_WORKER_BATCH_GRV_INTERVAL;
 
 	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
 	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX;
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 4f94391dd2..c914b13ae7 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -323,7 +323,9 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 		ASSERT_WE_THINK(false);
 
 		// if not simulation, kill the BM
-		bmData->iAmReplaced.sendError(e);
+		if (bmData->iAmReplaced.canBeSet()) {
+			bmData->iAmReplaced.sendError(e);
+		}
 		throw e;
 	}
 }
@@ -452,7 +454,9 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			    .error(e)
 			    .detail("Epoch", bmData->epoch);
 			ASSERT_WE_THINK(false);
-			bmData->iAmReplaced.sendError(e);
+			if (bmData->iAmReplaced.canBeSet()) {
+				bmData->iAmReplaced.sendError(e);
+			}
 			throw;
 		}
 
@@ -698,7 +702,9 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 					    .detail("Epoch", bmData->epoch)
 					    .detail("ClientRanges", results.size() - 1);
 					wait(delay(600));
-					bmData->iAmReplaced.sendError(internal_error());
+					if (bmData->iAmReplaced.canBeSet()) {
+						bmData->iAmReplaced.sendError(internal_error());
+					}
 					throw internal_error();
 				}
 
@@ -1041,6 +1047,7 @@ ACTOR Future<Void> deregisterBlobWorker(Reference<BlobManagerData> bmData, BlobW
 		try {
 			wait(checkManagerLock(tr, bmData));
 			Key blobWorkerListKey = blobWorkerListKeyFor(interf.id());
+			// FIXME: should be able to remove this conflict range
 			tr->addReadConflictRange(singleKeyRange(blobWorkerListKey));
 			tr->clear(blobWorkerListKey);
 
@@ -1298,7 +1305,9 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 				    .detail("Epoch", bmData->epoch);
 				ASSERT_WE_THINK(false);
 				// if not simulation, kill the BM
-				bmData->iAmReplaced.sendError(e);
+				if (bmData->iAmReplaced.canBeSet()) {
+					bmData->iAmReplaced.sendError(e);
+				}
 				throw e;
 			}
 		}
@@ -1343,7 +1352,9 @@ ACTOR Future<Void> monitorBlobWorker(Reference<BlobManagerData> bmData, BlobWork
 			    .detail("Epoch", bmData->epoch);
 			ASSERT_WE_THINK(false);
 			// if not simulation, kill the BM
-			bmData->iAmReplaced.sendError(e);
+			if (bmData->iAmReplaced.canBeSet()) {
+				bmData->iAmReplaced.sendError(e);
+			}
 			throw e;
 		}
 	}
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 2197885b6b..4ec0c18886 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -44,10 +44,17 @@
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/actorcompiler.h" // has to be last include
+#include "flow/network.h"
 
 #define BW_DEBUG false
 #define BW_REQUEST_DEBUG false
 
+/*
+ * The Blob Worker is a stateless role assigned a set of granules by the Blob Manager.
+ * It is responsible for managing the change feeds for those granules, and for consuming the mutations from those change
+ * feeds and writing them out as files to blob storage.
+ */
+
 struct GranuleStartState {
 	UID granuleID;
 	Version changeFeedStartVersion;
@@ -93,9 +100,6 @@ struct GranuleMetadata : NonCopyable, ReferenceCounted<GranuleMetadata> {
 
 	AssignBlobRangeRequest originalReq;
 
-	// TODO FOR DEBUGGING, REMOVE
-	Version waitForVersionReturned = invalidVersion;
-
 	void resume() {
 		if (resumeSnapshot.canBeSet()) {
 			resumeSnapshot.send(Void());
@@ -124,9 +128,6 @@ struct GranuleRangeMetadata {
 	GranuleRangeMetadata() : lastEpoch(0), lastSeqno(0) {}
 	GranuleRangeMetadata(int64_t epoch, int64_t seqno, Reference<GranuleMetadata> activeMetadata)
 	  : lastEpoch(epoch), lastSeqno(seqno), activeMetadata(activeMetadata) {}
-
-	// TODO REMOVE
-	// ~GranuleRangeMetadata() { printf("Destroying granule metadata\n"); }
 };
 
 // represents a previous version of a granule, and optionally the files that compose it
@@ -137,6 +138,7 @@ struct GranuleHistoryEntry : NonCopyable, ReferenceCounted<GranuleHistoryEntry>
 	Version endVersion; // version of the last delta file
 
 	// load files lazily, and allows for clearing old cold-queried files to save memory
+	// FIXME: add memory limit and evictor for old cached files
 	Future<GranuleFiles> files;
 
 	// FIXME: do skip pointers with single back-pointer and neighbor pointers
@@ -148,7 +150,6 @@ struct GranuleHistoryEntry : NonCopyable, ReferenceCounted<GranuleHistoryEntry>
 	  : range(range), granuleID(granuleID), startVersion(startVersion), endVersion(endVersion) {}
 };
 
-// TODO: add limit on active fdb initial snapshots we can do at once per worker (knob + flow lock)
 struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 	UID id;
 	Database db;
@@ -177,15 +178,15 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 
 	Promise<Void> doGRVCheck;
 	NotifiedVersion grvVersion;
+	Promise<Void> fatalError;
+
+	FlowLock initialSnapshotLock;
 
 	int changeFeedStreamReplyBufferSize = SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / 2;
 
-	BlobWorkerData(UID id, Database db) : id(id), db(db), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL) {}
-	~BlobWorkerData() {
-		if (BW_DEBUG) {
-			printf("Destroying BW %s data\n", id.toString().c_str());
-		}
-	}
+	BlobWorkerData(UID id, Database db)
+	  : id(id), db(db), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL),
+	    initialSnapshotLock(SERVER_KNOBS->BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM) {}
 
 	bool managerEpochOk(int64_t epoch) {
 		if (epoch < currentManagerEpoch) {
@@ -208,6 +209,18 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 	}
 };
 
+// serialize change feed key as UID bytes, to use 16 bytes on disk
+static Key granuleIDToCFKey(UID granuleID) {
+	BinaryWriter wr(Unversioned());
+	wr << granuleID;
+	return wr.toValue();
+}
+
+// parse change feed key back to UID, to be human-readable
+static UID cfKeyToGranuleID(Key cfKey) {
+	return BinaryReader::fromStringRef<UID>(cfKey, Unversioned());
+}
+
 // returns true if we can acquire it
 static void acquireGranuleLock(int64_t epoch, int64_t seqno, int64_t prevOwnerEpoch, int64_t prevOwnerSeqno) {
 	// returns true if our lock (E, S) >= (Eprev, Sprev)
@@ -225,11 +238,6 @@ static void acquireGranuleLock(int64_t epoch, int64_t seqno, int64_t prevOwnerEp
 
 static void checkGranuleLock(int64_t epoch, int64_t seqno, int64_t ownerEpoch, int64_t ownerSeqno) {
 	// sanity check - lock value should never go backwards because of acquireGranuleLock
-	/*
-	printf(
-	    "Checking granule lock: \n  mine: (%lld, %lld)\n  owner: (%lld, %lld)\n", epoch, seqno, ownerEpoch,
-	ownerSeqno);
-	    */
 	ASSERT(epoch <= ownerEpoch);
 	ASSERT(epoch < ownerEpoch || (epoch == ownerEpoch && seqno <= ownerSeqno));
 
@@ -258,6 +266,7 @@ ACTOR Future<Void> readAndCheckGranuleLock(Reference<ReadYourWritesTransaction>
 	checkGranuleLock(epoch, seqno, std::get<0>(currentOwner), std::get<1>(currentOwner));
 
 	// if we still own the lock, add a conflict range in case anybody else takes it over while we add this file
+	// FIXME: we don't need these conflict ranges
 	tr->addReadConflictRange(singleKeyRange(lockKey));
 
 	return Void();
@@ -321,7 +330,7 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 	state KeyRange currentRange = blobGranuleSplitKeyRangeFor(parentGranuleID);
 
 	state RangeResult totalState = wait(tr->getRange(currentRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2));
-	// TODO is this explicit conflit range necessary with the above read?
+	// FIXME: remove above conflict range?
 	tr->addWriteConflictRange(currentRange);
 	ASSERT_WE_THINK(!totalState.more && totalState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 	// maybe someone decreased the knob, we should gracefully handle it not in simulation
@@ -387,7 +396,7 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 			}
 
 			// FIXME: appears change feed destroy isn't working! ADD BACK
-			// wait(updateChangeFeed(tr, KeyRef(parentGranuleID.toString()), ChangeFeedStatus::CHANGE_FEED_DESTROY));
+			// wait(updateChangeFeed(tr, granuleIDToCFKey(parentGranuleID), ChangeFeedStatus::CHANGE_FEED_DESTROY));
 
 			Key oldGranuleLockKey = blobGranuleLockKeyFor(parentGranuleRange);
 			// FIXME: deleting granule lock can cause races where another granule with the same range starts way later
@@ -398,6 +407,7 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 
 			// tr->clear(singleKeyRange(oldGranuleLockKey));
 			tr->clear(currentRange);
+			TEST(true); // Granule split cleanup on last delta file persisted
 		} else {
 			tr->atomicOp(myStateKey, blobGranuleSplitValueFor(newState), MutationRef::SetVersionstampedValue);
 			if (newState == BlobGranuleSplitState::Assigned && currentState == BlobGranuleSplitState::Initialized &&
@@ -409,10 +419,13 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 					           parentGranuleID.toString().c_str());
 				}
 
-				wait(updateChangeFeed(tr, KeyRef(parentGranuleID.toString()), ChangeFeedStatus::CHANGE_FEED_STOP));
+				wait(updateChangeFeed(
+				    tr, KeyRef(granuleIDToCFKey(parentGranuleID)), ChangeFeedStatus::CHANGE_FEED_STOP));
 			}
+			TEST(true); // Granule split stopping change feed
 		}
 	} else if (BW_DEBUG) {
+		TEST(true); // Out of order granule split state updates ignored
 		fmt::print("Ignoring granule {0} split state from {1} {2} -> {3}\n",
 		           currentGranuleID.toString(),
 		           parentGranuleID.toString(),
@@ -481,7 +494,6 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 	state int numIterations = 0;
 	try {
 		// before updating FDB, wait for the delta file version to be committed and previous delta files to finish
-		// TODO fix file leak here on error pre-transaction.
 		wait(waitCommitted);
 		BlobFileIndex prev = wait(previousDeltaFileFuture);
 		wait(delay(0, TaskPriority::BlobWorkerUpdateFDB));
@@ -532,11 +544,13 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 		// commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB
 		// with file and cannot safely delete it.
 		if (numIterations > 0) {
+			TEST(true); // Granule potentially leaving orphaned delta file
 			throw e;
 		}
 		if (BW_DEBUG) {
 			fmt::print("deleting delta file {0} after error {1}\n", fname, e.name());
 		}
+		TEST(true); // Granule cleaning up delta file after error
 		++bwData->stats.s3DeleteReqs;
 		bwData->addActor.send(bwData->bstore->deleteFile(fname));
 		throw e;
@@ -583,18 +597,19 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 		           snapshot.size());
 	}
 
-	// TODO REMOVE sanity checks!
-	if (snapshot.size() > 0) {
-		ASSERT(keyRange.begin <= snapshot[0].key);
-		ASSERT(keyRange.end > snapshot[snapshot.size() - 1].key);
-	}
-	for (int i = 0; i < snapshot.size() - 1; i++) {
-		if (snapshot[i].key >= snapshot[i + 1].key) {
-			fmt::print("SORT ORDER VIOLATION IN SNAPSHOT FILE: {0}, {1}\n",
-			           snapshot[i].key.printable(),
-			           snapshot[i + 1].key.printable());
+	if (g_network->isSimulated()) {
+		if (snapshot.size() > 0) {
+			ASSERT(keyRange.begin <= snapshot[0].key);
+			ASSERT(keyRange.end > snapshot[snapshot.size() - 1].key);
+		}
+		for (int i = 0; i < snapshot.size() - 1; i++) {
+			if (snapshot[i].key >= snapshot[i + 1].key) {
+				fmt::print("SORT ORDER VIOLATION IN SNAPSHOT FILE: {0}, {1}\n",
+				           snapshot[i].key.printable(),
+				           snapshot[i + 1].key.printable());
+			}
+			ASSERT(snapshot[i].key < snapshot[i + 1].key);
 		}
-		ASSERT(snapshot[i].key < snapshot[i + 1].key);
 	}
 
 	state Value serialized = ObjectWriter::toValue(snapshot, Unversioned());
@@ -610,7 +625,6 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 	++bwData->stats.snapshotFilesWritten;
 	bwData->stats.snapshotBytesWritten += serializedSize;
 
-	// TODO: inject write error
 	wait(objectFile->append(serialized.begin(), serializedSize));
 	wait(objectFile->finish());
 
@@ -650,11 +664,13 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 		// commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB
 		// with file and cannot safely delete it.
 		if (numIterations > 0) {
+			TEST(true); // Granule potentially leaving orphaned snapshot file
 			throw e;
 		}
 		if (BW_DEBUG) {
 			fmt::print("deleting snapshot file {0} after error {1}\n", fname, e.name());
 		}
+		TEST(true); // Granule deleting snapshot file after error
 		++bwData->stats.s3DeleteReqs;
 		bwData->addActor.send(bwData->bstore->deleteFile(fname));
 		throw e;
@@ -684,6 +700,9 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 		           metadata->keyRange.begin.printable(),
 		           metadata->keyRange.end.printable());
 	}
+	wait(bwData->initialSnapshotLock.take());
+	state FlowLock::Releaser holdingDVL(bwData->initialSnapshotLock);
+
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
 	state int64_t bytesRead = 0;
 	state int retries = 0;
@@ -731,6 +750,7 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 			state Error err = e;
 			wait(tr->onError(e));
 			retries++;
+			TEST(true); // Granule initial snapshot failed
 			TraceEvent(SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id)
 			    .error(err)
 			    .detail("Granule", metadata->keyRange)
@@ -749,7 +769,6 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 
 // files might not be the current set of files in metadata, in the case of doing the initial snapshot of a granule that
 // was split.
-// FIXME: only pass metadata->keyRange
 ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
                                             Reference<GranuleMetadata> metadata,
                                             UID granuleID,
@@ -797,12 +816,6 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
 		           metadata->keyRange.begin.printable(),
 		           metadata->keyRange.end.printable(),
 		           version);
-
-		/*printf("  SnapshotFile:\n    %s\n", chunk.snapshotFile.get().toString().c_str());
-		printf("  DeltaFiles:\n");
-		for (auto& df : chunk.deltaFiles) {
-		    printf("    %s\n", df.toString().c_str());
-		}*/
 	}
 
 	loop {
@@ -819,8 +832,6 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
 			RangeResult newGranule =
 			    wait(readBlobGranule(chunk, metadata->keyRange, version, bwData->bstore, &bwData->stats));
 
-			// TODO: inject read error
-
 			bwData->stats.bytesReadFromS3ForCompaction += compactBytesRead;
 			rowsStream.send(std::move(newGranule));
 			rowsStream.sendError(end_of_stream());
@@ -873,8 +884,11 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 	metadata->resumeSnapshot.reset();
 	state int64_t statusEpoch = metadata->continueEpoch;
 	state int64_t statusSeqno = metadata->continueSeqno;
-	// TODO its own knob or something better? This is wrong in case of rollbacks
-	state bool writeHot = versionsSinceLastSnapshot <= SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS;
+
+	// If two snapshots happen without a split within a low time interval, this granule is "write-hot"
+	// FIXME: If a rollback happens, this could incorrectly identify a hot granule as not hot. This should be rare
+	// though and is just less efficient.
+	state bool writeHot = versionsSinceLastSnapshot <= SERVER_KNOBS->BG_HOT_SNAPSHOT_VERSIONS;
 	loop {
 		loop {
 			try {
@@ -897,6 +911,7 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 				if (e.code() == error_code_operation_cancelled) {
 					throw e;
 				}
+				TEST(true); // Blob worker re-sending split evaluation to manager after not error/not hearing back
 				// if we got broken promise while waiting, the old stream was killed, so we don't need to wait on
 				// change, just retry
 				if (e.code() == error_code_broken_promise) {
@@ -930,8 +945,6 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 	TraceEvent("BlobGranuleSnapshotFile", bwData->id)
 	    .detail("Granule", metadata->keyRange)
 	    .detail("Version", metadata->durableDeltaVersion.get());
-	// TODO: this could read from FDB instead if it knew there was a large range clear at the end or
-	// it knew the granule was small, or something
 
 	// wait for file updater to make sure that last delta file is in the metadata before
 	while (metadata->files.deltaFiles.empty() || metadata->files.deltaFiles.back().version < reSnapshotVersion) {
@@ -977,7 +990,6 @@ ACTOR Future<Void> handleCompletedDeltaFile(Reference<BlobWorkerData> bwData,
 }
 
 // if we get an i/o error updating files, or a rollback, reassign the granule to ourselves and start fresh
-// FIXME: is this the correct set of errors?
 static bool granuleCanRetry(const Error& e) {
 	switch (e.code()) {
 	case error_code_please_reboot:
@@ -1000,12 +1012,6 @@ struct InFlightFile {
 	  : future(future), version(version), bytes(bytes), snapshot(snapshot) {}
 };
 
-// TODO REMOVE once correctness clean
-#define DEBUG_BW_START_VERSION invalidVersion
-#define DEBUG_BW_END_VERSION invalidVersion
-#define DEBUG_BW_WAIT_VERSION invalidVersion
-#define DEBUG_BW_VERSION(v) DEBUG_BW_START_VERSION <= v&& v <= DEBUG_BW_END_VERSION
-
 static Version doGranuleRollback(Reference<GranuleMetadata> metadata,
                                  Version mutationVersion,
                                  Version rollbackVersion,
@@ -1024,6 +1030,7 @@ static Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 		for (auto& f : inFlightFiles) {
 			if (f.snapshot) {
 				if (f.version > rollbackVersion) {
+					TEST(true); // Granule rollback cancelling snapshot file
 					if (BW_DEBUG) {
 						fmt::print("[{0} - {1}) rollback cancelling snapshot file @ {2}\n",
 						           metadata->keyRange.begin.printable(),
@@ -1044,6 +1051,7 @@ static Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 						metadata->bytesInNewDeltaFiles -= f.bytes;
 					}
 					toPop++;
+					TEST(true); // Granule rollback cancelling delta file
 					if (BW_DEBUG) {
 						fmt::print("[{0} - {1}) rollback cancelling delta file @ {2}\n",
 						           metadata->keyRange.begin.printable(),
@@ -1093,6 +1101,7 @@ static Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 
 	} else {
 		// No pending delta files to discard, just in-memory mutations
+		TEST(true); // Granule rollback discarding in memory mutations
 
 		// FIXME: could binary search?
 		int mIdx = metadata->currentDeltas.size() - 1;
@@ -1135,44 +1144,18 @@ static Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 	return cfRollbackVersion;
 }
 
-#define DEBUG_WAIT_VERSION_COMMITTED false
-ACTOR Future<Void> waitOnCFVersion(Reference<GranuleMetadata> metadata,
-                                   Version original /*TODO REMOVE, just for debugging*/,
-                                   Version waitVersion) {
-	if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}:   waiting for {1} \n", original, waitVersion);
-	}
+ACTOR Future<Void> waitOnCFVersion(Reference<GranuleMetadata> metadata, Version waitVersion) {
 	loop {
 		try {
-			if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-				if (metadata->activeCFData.get().isValid()) {
-					fmt::print(
-					    "WVC {0}:     WAL (currently {1})\n", original, metadata->activeCFData.get()->getVersion());
-				} else {
-					fmt::print("WVC {0}:     invalid\n", original, metadata->activeCFData.get()->getVersion());
-				}
-			}
 			// if not valid, we're about to be cancelled anyway
 			state Future<Void> atLeast = metadata->activeCFData.get().isValid()
 			                                 ? metadata->activeCFData.get()->whenAtLeast(waitVersion)
 			                                 : Never();
 			choose {
-				when(wait(atLeast)) {
-					if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-						fmt::print("WVC {0}:   got at least {1} \n", original, waitVersion);
-					}
-					break;
-				}
-				when(wait(metadata->activeCFData.onChange())) {
-					if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-						fmt::print("WVC {0}:     cfOnChange \n", original);
-					}
-				}
+				when(wait(atLeast)) { break; }
+				when(wait(metadata->activeCFData.onChange())) {}
 			}
 		} catch (Error& e) {
-			if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-				fmt::print("WVC {0}:   got error {1} \n", original, e.name());
-			}
 			if (e.code() == error_code_operation_cancelled || e.code() == error_code_change_feed_popped) {
 				throw e;
 			}
@@ -1180,25 +1163,10 @@ ACTOR Future<Void> waitOnCFVersion(Reference<GranuleMetadata> metadata,
 			// if waiting on a parent granule change feed and we change to the child, the parent will get end_of_stream,
 			// which could cause this waiting whenAtLeast to get change_feed_cancelled. We should simply retry and wait
 			// a bit, as blobGranuleUpdateFiles will switch to the new change feed
-			if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-				if (BW_DEBUG) {
-					fmt::print("WVC {0}:   unexpected error {1}\n", original, e.name());
-				}
-				throw e;
-			}
 			wait(delay(0.05));
 		}
 	}
 
-	if (DEBUG_BW_VERSION(original) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}:   got \n", original);
-	}
-
-	// sanity check to make sure whenAtLeast didn't return early
-	if (waitVersion > metadata->waitForVersionReturned) {
-		metadata->waitForVersionReturned = waitVersion;
-	}
-
 	// stop after change feed callback
 	wait(delay(0, TaskPriority::BlobWorkerReadChangeFeed));
 
@@ -1208,17 +1176,7 @@ ACTOR Future<Void> waitOnCFVersion(Reference<GranuleMetadata> metadata,
 ACTOR Future<Void> waitCommittedGrv(Reference<BlobWorkerData> bwData,
                                     Reference<GranuleMetadata> metadata,
                                     Version version) {
-	if (DEBUG_BW_VERSION(version) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}:   grv start\n", version);
-	}
-	// TODO REMOVE debugs
 	if (version > bwData->grvVersion.get()) {
-		if (DEBUG_BW_VERSION(version) && DEBUG_WAIT_VERSION_COMMITTED) {
-			fmt::print("WVC {0}:     getting grv\n", version);
-		}
-		/*if (BW_DEBUG) {
-		    fmt::print("waitVersionCommitted waiting {0}\n", version);
-		}*/
 		// this order is important, since we need to register a waiter on the notified version before waking the GRV
 		// actor
 		Future<Void> grvAtLeast = bwData->grvVersion.whenAtLeast(version);
@@ -1230,10 +1188,7 @@ ACTOR Future<Void> waitCommittedGrv(Reference<BlobWorkerData> bwData,
 	}
 
 	Version grvVersion = bwData->grvVersion.get();
-	if (DEBUG_BW_VERSION(version) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}:     got grv\n", version);
-	}
-	wait(waitOnCFVersion(metadata, version, grvVersion));
+	wait(waitOnCFVersion(metadata, grvVersion));
 	return Void();
 }
 
@@ -1242,14 +1197,8 @@ ACTOR Future<Void> waitVersionCommitted(Reference<BlobWorkerData> bwData,
                                         Version version) {
 	// If GRV is way in the future, we know we can't roll back more than 5 seconds (or whatever this knob is set to)
 	// worth of versions
-	if (DEBUG_BW_VERSION(version) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}: starting\n", version);
-	}
 	wait(waitCommittedGrv(bwData, metadata, version) ||
-	     waitOnCFVersion(metadata, version, version + SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS));
-	if (DEBUG_BW_VERSION(version) && DEBUG_WAIT_VERSION_COMMITTED) {
-		fmt::print("WVC {0}: done\n", version);
-	}
+	     waitOnCFVersion(metadata, version + SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS));
 	if (version > metadata->knownCommittedVersion) {
 		metadata->knownCommittedVersion = version;
 	}
@@ -1258,7 +1207,6 @@ ACTOR Future<Void> waitVersionCommitted(Reference<BlobWorkerData> bwData,
 
 // updater for a single granule
 // TODO: this is getting kind of large. Should try to split out this actor if it continues to grow?
-// FIXME: handle errors here (forward errors)
 ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
                                           Reference<GranuleMetadata> metadata,
                                           Future<GranuleStartState> assignFuture) {
@@ -1288,9 +1236,9 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 		wait(delay(0, TaskPriority::BlobWorkerUpdateStorage));
 
-		cfKey = StringRef(startState.granuleID.toString());
+		cfKey = granuleIDToCFKey(startState.granuleID);
 		if (startState.parentGranule.present()) {
-			oldCFKey = StringRef(startState.parentGranule.get().second.toString());
+			oldCFKey = granuleIDToCFKey(startState.parentGranule.get().second);
 		}
 
 		if (BW_DEBUG) {
@@ -1329,6 +1277,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		}
 
 		if (!startState.doSnapshot) {
+			TEST(true); // Granule moved without split
 			startVersion = startState.previousDurableVersion;
 			ASSERT(!metadata->files.snapshotFiles.empty());
 			metadata->pendingSnapshotVersion = metadata->files.snapshotFiles.back().version;
@@ -1363,7 +1312,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		metadata->bufferedDeltaVersion = startVersion;
 		metadata->knownCommittedVersion = startVersion;
 
-		Reference<ChangeFeedData> newCFData = makeReference<ChangeFeedData>();
+		Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
 
 		if (startState.parentGranule.present() && startVersion < startState.changeFeedStartVersion) {
 			// read from parent change feed up until our new change feed is started
@@ -1374,7 +1323,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			// this actor
 			readOldChangeFeed = true;
 
-			oldChangeFeedFuture = bwData->db->getChangeFeedStream(newCFData,
+			oldChangeFeedFuture = bwData->db->getChangeFeedStream(cfData,
 			                                                      oldCFKey.get(),
 			                                                      startVersion + 1,
 			                                                      startState.changeFeedStartVersion,
@@ -1384,7 +1333,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 		} else {
 			readOldChangeFeed = false;
-			changeFeedFuture = bwData->db->getChangeFeedStream(newCFData,
+			changeFeedFuture = bwData->db->getChangeFeedStream(cfData,
 			                                                   cfKey,
 			                                                   startVersion + 1,
 			                                                   MAX_VERSION,
@@ -1395,7 +1344,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 		// Start actors BEFORE setting new change feed data to ensure the change feed data is properly initialized by
 		// the client
-		metadata->activeCFData.set(newCFData);
+		metadata->activeCFData.set(cfData);
 
 		ASSERT(metadata->readable.canBeSet());
 		metadata->readable.send(Void());
@@ -1441,22 +1390,12 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 			state Standalone<VectorRef<MutationsAndVersionRef>> mutations;
 			try {
-				/*if (DEBUG_BW_VERSION(metadata->bufferedDeltaVersion)) {
-				    fmt::print("BW waiting mutations after ({0})\n", metadata->bufferedDeltaVersion);
-				}*/
 				// Even if there are no new mutations, there still might be readers waiting on durableDeltaVersion
 				// to advance. We need to check whether any outstanding files have finished so we don't wait on
 				// mutations forever
 				choose {
 					when(Standalone<VectorRef<MutationsAndVersionRef>> _mutations =
 					         waitNext(metadata->activeCFData.get()->mutations.getFuture())) {
-						/*if (DEBUG_BW_VERSION(metadata->bufferedDeltaVersion)) {
-						    fmt::print("BW got mutations after ({0}): {1} - {2} ({3})\n",
-						               metadata->bufferedDeltaVersion,
-						               _mutations.front().version,
-						               _mutations.back().version,
-						               _mutations.size());
-						}*/
 						mutations = _mutations;
 						ASSERT(!mutations.empty());
 						if (readOldChangeFeed) {
@@ -1475,28 +1414,10 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 						}
 						ASSERT(mutations.front().version > metadata->bufferedDeltaVersion);
 
-						// if we just got mutations, we haven't buffered them yet, so waitForVersion can't have returned
-						// this version yet
-						if (mutations.front().version <= metadata->waitForVersionReturned) {
-							fmt::print("ERROR: WaitForVersion returned early for granule [{0} - {1}). "
-							           "waitForVersionReturned={2}, mutationVersion={3} !!!\n",
-							           metadata->keyRange.begin.printable(),
-							           metadata->keyRange.end.printable(),
-							           metadata->waitForVersionReturned,
-							           mutations.front().version);
-						}
-						ASSERT(mutations.front().version > metadata->waitForVersionReturned);
-
 						// If this assert trips we should have gotten change_feed_popped from SS and didn't
 						ASSERT(mutations.front().version >= metadata->activeCFData.get()->popVersion);
 					}
-					when(wait(inFlightFiles.empty() ? Never() : success(inFlightFiles.front().future))) {
-						//  TODO REMOVE
-						/*if (DEBUG_BW_VERSION(metadata->bufferedDeltaVersion)) {
-						    fmt::print("BW got file before waiting for mutations after {0}\n",
-						               metadata->bufferedDeltaVersion);
-						}*/
-					}
+					when(wait(inFlightFiles.empty() ? Never() : success(inFlightFiles.front().future))) {}
 				}
 			} catch (Error& e) {
 				// only error we should expect here is when we finish consuming old change feed
@@ -1517,9 +1438,9 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					           metadata->bufferedDeltaVersion);
 				}
 
-				Reference<ChangeFeedData> newCFData = makeReference<ChangeFeedData>();
+				Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
 
-				changeFeedFuture = bwData->db->getChangeFeedStream(newCFData,
+				changeFeedFuture = bwData->db->getChangeFeedStream(cfData,
 				                                                   cfKey,
 				                                                   startState.changeFeedStartVersion,
 				                                                   MAX_VERSION,
@@ -1529,7 +1450,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 				// Start actors BEFORE setting new change feed data to ensure the change feed data is properly
 				// initialized by the client
-				metadata->activeCFData.set(newCFData);
+				metadata->activeCFData.set(cfData);
 			}
 
 			// process mutations
@@ -1557,20 +1478,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 							ASSERT(rollbackVersion >= metadata->durableDeltaVersion.get());
 
 							if (!rollbacksInProgress.empty()) {
-								// TODO REMOVE, for debugging
-								if (rollbacksInProgress.front().first != rollbackVersion) {
-									fmt::print("Found out of order rollbacks! Current in progress: {0}, mutation "
-									           "version: {1}\n",
-									           rollbacksInProgress.front().first,
-									           rollbackVersion);
-								}
 								ASSERT(rollbacksInProgress.front().first == rollbackVersion);
-								if (rollbacksInProgress.front().first != rollbackVersion) {
-									fmt::print("Found out of order rollbacks! Current in progress: {0}, rollback "
-									           "version: {1}\n",
-									           rollbacksInProgress.front().second,
-									           deltas.version);
-								}
 								ASSERT(rollbacksInProgress.front().second == deltas.version);
 								if (BW_DEBUG) {
 									fmt::print("Passed rollback {0} -> {1}\n", deltas.version, rollbackVersion);
@@ -1586,6 +1494,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 								     metadata->currentDeltas.back().version <= rollbackVersion)) {
 
 									if (BW_DEBUG) {
+										TEST(true); // Granule ignoring rollback
 										fmt::print(
 										    "Granule [{0} - {1}) on BW {2} skipping rollback {3} -> {4} completely\n",
 										    metadata->keyRange.begin.printable().c_str(),
@@ -1600,6 +1509,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 									// stream.
 									rollbacksCompleted.push_back(std::pair(rollbackVersion, deltas.version));
 								} else {
+									TEST(true); // Granule processing rollback
 									if (BW_DEBUG) {
 										fmt::print("[{0} - {1}) on BW {2} ROLLBACK @ {3} -> {4}\n",
 										           metadata->keyRange.begin.printable(),
@@ -1620,15 +1530,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 									                                              rollbacksInProgress,
 									                                              rollbacksCompleted);
 
-									// Reset change feeds to cfRollbackVersion
-									if (cfRollbackVersion < metadata->waitForVersionReturned) {
-										fmt::print("Rollback resetting waitForVersionReturned {0} -> {1}\n",
-										           metadata->waitForVersionReturned,
-										           cfRollbackVersion);
-										metadata->waitForVersionReturned = cfRollbackVersion;
-									}
-
-									Reference<ChangeFeedData> newCFData = makeReference<ChangeFeedData>();
+									Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
 
 									if (!readOldChangeFeed && cfRollbackVersion < startState.changeFeedStartVersion) {
 										// It isn't possible to roll back across the parent/child feed boundary, but as
@@ -1645,7 +1547,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 									if (readOldChangeFeed) {
 										ASSERT(cfRollbackVersion < startState.changeFeedStartVersion);
 										oldChangeFeedFuture =
-										    bwData->db->getChangeFeedStream(newCFData,
+										    bwData->db->getChangeFeedStream(cfData,
 										                                    oldCFKey.get(),
 										                                    cfRollbackVersion + 1,
 										                                    startState.changeFeedStartVersion,
@@ -1662,7 +1564,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 										ASSERT(cfRollbackVersion >= startState.changeFeedStartVersion);
 
 										changeFeedFuture =
-										    bwData->db->getChangeFeedStream(newCFData,
+										    bwData->db->getChangeFeedStream(cfData,
 										                                    cfKey,
 										                                    cfRollbackVersion + 1,
 										                                    MAX_VERSION,
@@ -1673,7 +1575,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 									// Start actors BEFORE setting new change feed data to ensure the change feed data
 									// is properly initialized by the client
-									metadata->activeCFData.set(newCFData);
+									metadata->activeCFData.set(cfData);
 
 									justDidRollback = true;
 									break;
@@ -1681,6 +1583,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 							}
 						} else if (!rollbacksInProgress.empty() && rollbacksInProgress.front().first < deltas.version &&
 						           rollbacksInProgress.front().second > deltas.version) {
+							TEST(true); // Granule skipping mutations b/c prior rollback
 							if (BW_DEBUG) {
 								fmt::print("Skipping mutations @ {} b/c prior rollback\n", deltas.version);
 							}
@@ -1695,9 +1598,6 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 								    .detail("ChangeFeedID", readOldChangeFeed ? oldCFKey.get() : cfKey)
 								    .detail("OldChangeFeed", readOldChangeFeed ? "T" : "F");
 							}
-							if (DEBUG_BW_VERSION(deltas.version)) {
-								fmt::print("BWB {0}: ({1})\n", deltas.version, deltas.mutations.size());
-							}
 							metadata->currentDeltas.push_back_deep(metadata->currentDeltas.arena(), deltas);
 
 							processedAnyMutations = true;
@@ -1845,6 +1745,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 							idx++;
 						}
 						while (waitIdx > 0) {
+							TEST(true); // Granule blocking on previous snapshot
 							// TODO don't duplicate code
 							BlobFileIndex completedFile = wait(inFlightFiles.front().future);
 							if (inFlightFiles.front().snapshot) {
@@ -1877,6 +1778,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					// queue too many files in parallel, and slow down change feed consuming to let file writing
 					// catch up
 
+					TEST(true); // Granule processing long tail of old change feed
 					if (inFlightFiles.size() > 10 && inFlightFiles.front().version <= metadata->knownCommittedVersion) {
 						if (BW_DEBUG) {
 							fmt::print("[{0} - {1}) Waiting on delta file b/c old change feed\n",
@@ -1893,14 +1795,6 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			}
 		}
 	} catch (Error& e) {
-		// TODO REMOVE
-		if (BW_DEBUG) {
-			fmt::print("BGUF {0} [{1} - {2}) got error {3}\n",
-			           startState.granuleID.toString(),
-			           metadata->keyRange.begin.printable(),
-			           metadata->keyRange.end.printable(),
-			           e.name());
-		}
 		// Free last change feed data
 		metadata->activeCFData.set(Reference<ChangeFeedData>());
 
@@ -1915,32 +1809,47 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		if (e.code() == error_code_granule_assignment_conflict) {
 			TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id).detail("Granule", metadata->keyRange);
 			return Void();
-		} else {
-			++bwData->stats.granuleUpdateErrors;
-			if (BW_DEBUG) {
-				fmt::print("Granule file updater for [{0} - {1}) got error {2}, exiting\n",
-				           metadata->keyRange.begin.printable(),
-				           metadata->keyRange.end.printable(),
-				           e.name());
+		}
+		++bwData->stats.granuleUpdateErrors;
+		if (BW_DEBUG) {
+			fmt::print("Granule file updater for [{0} - {1}) got error {2}, exiting\n",
+			           metadata->keyRange.begin.printable(),
+			           metadata->keyRange.end.printable(),
+			           e.name());
+		}
+
+		if (granuleCanRetry(e)) {
+			TEST(true); // Granule close and re-open on error
+			TraceEvent("GranuleFileUpdaterRetriableError", bwData->id)
+			    .error(e)
+			    .detail("Granule", metadata->keyRange)
+			    .detail("GranuleID", startState.granuleID);
+			// explicitly cancel all outstanding write futures BEFORE updating promise stream, to ensure they
+			// can't update files after the re-assigned granule acquires the lock
+			// do it backwards though because future depends on previous one, so it could cause a cascade
+			for (int i = inFlightFiles.size() - 1; i >= 0; i--) {
+				inFlightFiles[i].future.cancel();
 			}
-			TraceEvent(SevWarn, "GranuleFileUpdaterError", bwData->id).error(e).detail("Granule", metadata->keyRange);
 
-			if (granuleCanRetry(e)) {
-				// explicitly cancel all outstanding write futures BEFORE updating promise stream, to ensure they
-				// can't update files after the re-assigned granule acquires the lock
-				// do it backwards though because future depends on previous one, so it could cause a cascade
-				for (int i = inFlightFiles.size() - 1; i >= 0; i--) {
-					inFlightFiles[i].future.cancel();
-				}
+			// if we retry and re-open, we need to use a normal request (no continue) and update the
+			// seqno
+			metadata->originalReq.managerEpoch = metadata->continueEpoch;
+			metadata->originalReq.managerSeqno = metadata->continueSeqno;
+			metadata->originalReq.type = AssignRequestType::Normal;
 
-				// if we retry and re-open, we need to use a normal request (no continue) and update the
-				// seqno
-				metadata->originalReq.managerEpoch = metadata->continueEpoch;
-				metadata->originalReq.managerSeqno = metadata->continueSeqno;
-				metadata->originalReq.type = AssignRequestType::Normal;
+			bwData->granuleUpdateErrors.send(metadata->originalReq);
+			throw e;
+		}
 
-				bwData->granuleUpdateErrors.send(metadata->originalReq);
-			}
+		TraceEvent(SevError, "GranuleFileUpdaterUnexpectedError", bwData->id)
+		    .error(e)
+		    .detail("Granule", metadata->keyRange)
+		    .detail("GranuleID", startState.granuleID);
+		ASSERT_WE_THINK(false);
+
+		// if not simulation, kill the BW
+		if (bwData->fatalError.canBeSet()) {
+			bwData->fatalError.sendError(e);
 		}
 		throw e;
 	}
@@ -1997,10 +1906,6 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 
 			if (!historyEntryStack.empty()) {
 				Version oldestStartVersion = historyEntryStack.back()->startVersion;
-				// TODO REMOVE eventually, for debugging
-				if (stopVersion != oldestStartVersion && stopVersion != invalidVersion) {
-					fmt::print("Finished, stopVersion={0}, curHistory.version={1}\n", stopVersion, oldestStartVersion);
-				}
 				ASSERT(stopVersion == oldestStartVersion || stopVersion == invalidVersion);
 			} else {
 				ASSERT(stopVersion == invalidVersion);
@@ -2016,6 +1921,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 
 			int i = historyEntryStack.size() - 1;
 			while (i >= 0 && historyEntryStack[i]->startVersion <= stopVersion) {
+				TEST(true); // Granule skipping history entries loaded by parallel reader
 				i--;
 			}
 			int skipped = historyEntryStack.size() - 1 - i;
@@ -2050,11 +1956,14 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 		if (e.code() == error_code_granule_assignment_conflict) {
 			return Void();
 		}
-		if (BW_DEBUG) {
-			fmt::print("Loading blob granule history got unexpected error {}\n", e.name());
+		// SplitStorageMetrics explicitly has a SevError if it gets an error, so no errors should propagate here
+		TraceEvent(SevError, "BlobWorkerUnexpectedErrorLoadGranuleHistory", bwData->id).error(e);
+		ASSERT_WE_THINK(false);
+
+		// if not simulation, kill the BW
+		if (bwData->fatalError.canBeSet()) {
+			bwData->fatalError.sendError(e);
 		}
-		// TODO this should never happen?
-		ASSERT(false);
 		throw e;
 	}
 }
@@ -2083,52 +1992,19 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 
 	ASSERT(metadata->activeCFData.get().isValid());
 
-	if (v == DEBUG_BW_WAIT_VERSION) {
-		fmt::print("{0}) [{1} - {2}) waiting for {3}\n  readable:{4}\n  bufferedDelta={5}\n  pendingDelta={6}\n  "
-		           "durableDelta={7}\n  pendingSnapshot={8}\n  durableSnapshot={9}\n",
-		           v,
-		           metadata->keyRange.begin.printable().c_str(),
-		           metadata->keyRange.end.printable().c_str(),
-		           v,
-		           metadata->readable.isSet() ? "T" : "F",
-		           metadata->activeCFData.get()->getVersion(),
-		           metadata->pendingDeltaVersion,
-		           metadata->durableDeltaVersion.get(),
-		           metadata->pendingSnapshotVersion,
-		           metadata->durableSnapshotVersion.get());
-	}
-
 	if (v <= metadata->activeCFData.get()->getVersion() &&
 	    (v <= metadata->durableDeltaVersion.get() ||
 	     metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion) &&
 	    (v <= metadata->durableSnapshotVersion.get() ||
 	     metadata->durableSnapshotVersion.get() == metadata->pendingSnapshotVersion)) {
-		// TODO REMOVE debugging
-		if (v > metadata->waitForVersionReturned) {
-			metadata->waitForVersionReturned = v;
-		}
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) already done\n", v);
-		}
+		TEST(true); // Granule read not waiting
 		return Void();
 	}
 
 	// wait for change feed version to catch up to ensure we have all data
 	if (metadata->activeCFData.get()->getVersion() < v) {
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) waiting for CF version (currently {1})\n", v, metadata->activeCFData.get()->getVersion());
-		}
-
 		wait(metadata->activeCFData.get()->whenAtLeast(v));
 		ASSERT(metadata->activeCFData.get()->getVersion() >= v);
-
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) got CF version {1}\n", v, metadata->activeCFData.get()->getVersion());
-		}
-		// TODO REMOVE debugging
-		if (v > metadata->waitForVersionReturned) {
-			metadata->waitForVersionReturned = v;
-		}
 	}
 
 	// wait for any pending delta and snapshot files as of the moment the change feed version caught up.
@@ -2138,31 +2014,17 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// If there are mutations that are no longer buffered but have not been
 	// persisted to a delta file that are necessary for the query, wait for them
 	if (pendingDeltaV > metadata->durableDeltaVersion.get() && v > metadata->durableDeltaVersion.get()) {
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) waiting for DDV {1} < {2}\n", v, metadata->durableDeltaVersion.get(), pendingDeltaV);
-		}
-
+		TEST(true); // Granule read waiting for pending delta
 		wait(metadata->durableDeltaVersion.whenAtLeast(pendingDeltaV));
 		ASSERT(metadata->durableDeltaVersion.get() >= pendingDeltaV);
-
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) got DDV {1} >= {2}\n", v, metadata->durableDeltaVersion.get(), pendingDeltaV);
-		}
 	}
 
 	// This isn't strictly needed, but if we're in the process of re-snapshotting, we'd likely rather
 	// return that snapshot file than the previous snapshot file and all its delta files.
 	if (pendingSnapshotV > metadata->durableSnapshotVersion.get() && v > metadata->durableSnapshotVersion.get()) {
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) waiting for DSV {1} < {2}\n", v, metadata->durableSnapshotVersion.get(), pendingSnapshotV);
-		}
-
+		TEST(true); // Granule read waiting for pending snapshot
 		wait(metadata->durableSnapshotVersion.whenAtLeast(pendingSnapshotV));
 		ASSERT(metadata->durableSnapshotVersion.get() >= pendingSnapshotV);
-
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) got DSV {1} >= {2}\n", v, metadata->durableSnapshotVersion.get(), pendingSnapshotV);
-		}
 	}
 
 	// There is a race here - we wait for pending delta files before this to finish, but while we do, we
@@ -2170,27 +2032,16 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// file instead of in memory mutations, so we wait for that delta file to complete
 
 	if (metadata->pendingDeltaVersion >= v) {
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) waiting for DDV again {1} < {2}\n", v, metadata->durableDeltaVersion.get(), v);
-		}
-
+		TEST(true); // Granule mutations flushed while waiting for files to complete
 		wait(metadata->durableDeltaVersion.whenAtLeast(v));
 		ASSERT(metadata->durableDeltaVersion.get() >= v);
-
-		if (v == DEBUG_BW_WAIT_VERSION) {
-			fmt::print("{0}) got DDV again {1} >= {2}\n", v, metadata->durableDeltaVersion.get(), v);
-		}
-	}
-
-	if (v == DEBUG_BW_WAIT_VERSION) {
-		fmt::print("{0}) done\n", v);
 	}
 
 	return Void();
 }
 
 ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, BlobGranuleFileRequest req) {
-	if (BW_REQUEST_DEBUG || DEBUG_BW_WAIT_VERSION == req.readVersion) {
+	if (BW_REQUEST_DEBUG) {
 		fmt::print("BW {0} processing blobGranuleFileRequest for range [{1} - {2}) @ {3}\n",
 		           bwData->id.toString(),
 		           req.keyRange.begin.printable(),
@@ -2212,7 +2063,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 		for (auto& r : checkRanges) {
 			bool isValid = r.value().activeMetadata.isValid();
 			if (lastRangeEnd < r.begin() || !isValid) {
-				if (BW_REQUEST_DEBUG || DEBUG_BW_WAIT_VERSION == req.readVersion) {
+				if (BW_REQUEST_DEBUG) {
 					fmt::print("No {0} blob data for [{1} - {2}) in request range [{3} - {4}), skipping request\n",
 					           isValid ? "" : "valid",
 					           lastRangeEnd.printable(),
@@ -2227,7 +2078,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 			lastRangeEnd = r.end();
 		}
 		if (lastRangeEnd < req.keyRange.end) {
-			if (BW_REQUEST_DEBUG || DEBUG_BW_WAIT_VERSION == req.readVersion) {
+			if (BW_REQUEST_DEBUG) {
 				fmt::print("No blob data for [{0} - {1}) in request range [{2} - {3}), skipping request\n",
 				           lastRangeEnd.printable(),
 				           req.keyRange.end.printable(),
@@ -2263,6 +2114,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 			state GranuleFiles chunkFiles;
 
 			if (metadata->initialSnapshotVersion > req.readVersion) {
+				TEST(true); // Granule Time Travel Read
 				// this is a time travel query, find previous granule
 				if (metadata->historyLoaded.canBeSet()) {
 					choose {
@@ -2279,20 +2131,6 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				// FIXME: use skip pointers here
 				Version expectedEndVersion = metadata->initialSnapshotVersion;
 				if (cur.isValid()) {
-					// TODO REMOVE, useful for debugging for now
-					if (cur->endVersion != expectedEndVersion) {
-						fmt::print("Active granule [{0} - {1}) does not have history ancestor!!. Start is {2}, "
-						           "ancestor is [{3} - {4}) ({5}) V[{6} - {7}). SearchKey={8}\n",
-						           metadata->keyRange.begin.printable(),
-						           metadata->keyRange.end.printable(),
-						           expectedEndVersion,
-						           cur->range.begin.printable(),
-						           cur->range.end.printable(),
-						           cur->granuleID.toString(),
-						           cur->startVersion,
-						           cur->endVersion,
-						           historySearchKey.printable());
-					}
 					ASSERT(cur->endVersion == expectedEndVersion);
 				}
 				while (cur.isValid() && req.readVersion < cur->startVersion) {
@@ -2307,7 +2145,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 					throw blob_granule_transaction_too_old();
 				}
 
-				if (BW_REQUEST_DEBUG || DEBUG_BW_WAIT_VERSION == req.readVersion) {
+				if (BW_REQUEST_DEBUG) {
 					fmt::print("[{0} - {1}) @ {2} time traveled back to {3} [{4} - {5}) @ [{6} - {7})\n",
 					           req.keyRange.begin.printable(),
 					           req.keyRange.end.printable(),
@@ -2335,25 +2173,10 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 
 				ASSERT(!chunkFiles.snapshotFiles.empty());
 				ASSERT(!chunkFiles.deltaFiles.empty());
-				// TODO remove eventually, for help debugging asserts
-				if (chunkFiles.deltaFiles.back().version <= req.readVersion ||
-				    chunkFiles.snapshotFiles.front().version > req.readVersion) {
-					fmt::print("Time Travel read version {0} out of bounds!\n  current granule initial version: {1}\n  "
-					           "snapshot files ({2}):\n",
-					           req.readVersion,
-					           metadata->initialSnapshotVersion,
-					           chunkFiles.snapshotFiles.size());
-					for (auto& f : chunkFiles.snapshotFiles) {
-						fmt::print("    {0}}\n", f.version);
-					}
-					fmt::print("  delta files {0}:\n", chunkFiles.deltaFiles.size());
-					for (auto& f : chunkFiles.deltaFiles) {
-						fmt::print("    {0}\n", f.version);
-					}
-				}
 				ASSERT(chunkFiles.deltaFiles.back().version > req.readVersion);
 				ASSERT(chunkFiles.snapshotFiles.front().version <= req.readVersion);
 			} else {
+				TEST(true); // Granule Active Read
 				// this is an active granule query
 				loop {
 					if (!metadata->activeCFData.get().isValid() || !metadata->cancelled.canBeSet()) {
@@ -2373,18 +2196,20 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 							when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
 						}
 					} catch (Error& e) {
-						// we can get change feed cancelled from whenAtLeast. This is effectively
+						// We can get change feed cancelled from whenAtLeast. This means the change feed may retry, or
+						// may be cancelled. Wait a bit and try again to see
 						if (e.code() == error_code_change_feed_popped) {
+							TEST(true); // Change feed popped while read waiting
 							throw wrong_shard_server();
 						}
 						if (e.code() != error_code_change_feed_cancelled) {
 							throw e;
 						}
+						TEST(true); // Change feed switched while read waiting
 						// wait 1ms and try again
 						wait(delay(0.001));
 					}
-					if ((BW_REQUEST_DEBUG || DEBUG_BW_WAIT_VERSION == req.readVersion) &&
-					    metadata->activeCFData.get().isValid()) {
+					if ((BW_REQUEST_DEBUG) && metadata->activeCFData.get().isValid()) {
 						fmt::print("{0} - {1}) @ {2} hit CF change, restarting waitForVersion\n",
 						           req.keyRange.begin.printable().c_str(),
 						           req.keyRange.end.printable().c_str(),
@@ -2540,6 +2365,7 @@ ACTOR Future<Void> handleBlobGranuleFileRequest(Reference<BlobWorkerData> bwData
 	return Void();
 }
 
+// FIXME: move this up by other granule state stuff like BGUF
 ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, AssignBlobRangeRequest req) {
 	ASSERT(req.type != AssignRequestType::Continue);
 	state Transaction tr(bwData->db);
@@ -2561,12 +2387,13 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 			info.changeFeedStartVersion = invalidVersion;
 
 			state Future<Optional<Value>> fLockValue = tr.get(lockKey);
-			state Future<Optional<GranuleHistory>> fHistory = getLatestGranuleHistory(&tr, req.keyRange);
+			Future<Optional<GranuleHistory>> fHistory = getLatestGranuleHistory(&tr, req.keyRange);
 			Optional<GranuleHistory> history = wait(fHistory);
 			info.history = history;
 			Optional<Value> prevLockValue = wait(fLockValue);
 			state bool hasPrevOwner = prevLockValue.present();
 			if (hasPrevOwner) {
+				TEST(true); // Granule open found previous owner
 				std::tuple<int64_t, int64_t, UID> prevOwner = decodeBlobGranuleLockValue(prevLockValue.get());
 				acquireGranuleLock(req.managerEpoch, req.managerSeqno, std::get<0>(prevOwner), std::get<1>(prevOwner));
 				info.granuleID = std::get<2>(prevOwner);
@@ -2601,9 +2428,6 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 				info.changeFeedStartVersion = info.previousDurableVersion;
 			} else {
 				// else we are first, no need to check for owner conflict
-				// FIXME: use actual 16 bytes of UID instead of converting it to 32 character string and
-				// then that to bytes
-
 				if (info.history.present()) {
 					// if this granule is derived from a split or merge, this history entry is already
 					// present (written by the blob manager)
@@ -2614,7 +2438,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 					info.granuleID = deterministicRandom()->randomUniqueID();
 				}
 				wait(updateChangeFeed(
-				    &tr, StringRef(info.granuleID.toString()), ChangeFeedStatus::CHANGE_FEED_CREATE, req.keyRange));
+				    &tr, granuleIDToCFKey(info.granuleID), ChangeFeedStatus::CHANGE_FEED_CREATE, req.keyRange));
 				info.doSnapshot = true;
 				info.previousDurableVersion = invalidVersion;
 			}
@@ -2626,6 +2450,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 			// ret.previousChangeFeedId, and the previous durable version will come from the previous
 			// granules
 			if (info.history.present() && info.history.get().value.parentGranules.size() > 0) {
+				TEST(true); // Granule open found parent
 				// TODO change this for merge
 				ASSERT(info.history.get().value.parentGranules.size() == 1);
 				state KeyRange parentGranuleRange = info.history.get().value.parentGranules[0].first;
@@ -2650,10 +2475,12 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 					}
 
 					if (granuleSplitState.first == BlobGranuleSplitState::Assigned) {
+						TEST(true); // Granule open found granule in assign state
 						// was already assigned, use change feed start version
 						ASSERT(granuleSplitState.second > 0);
 						info.changeFeedStartVersion = granuleSplitState.second;
 					} else if (granuleSplitState.first == BlobGranuleSplitState::Initialized) {
+						TEST(true); // Granule open found granule in initialized state
 						wait(updateGranuleSplitState(&tr,
 						                             info.parentGranule.get().first,
 						                             info.parentGranule.get().second,
@@ -2662,6 +2489,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 						// change feed was created as part of this transaction, changeFeedStartVersion
 						// will be set later
 					} else {
+						TEST(true); // Granule open found granule in done state
 						// this sub-granule is done splitting, no need for split logic.
 						info.parentGranule.reset();
 					}
@@ -2763,8 +2591,6 @@ static bool changeBlobRange(Reference<BlobWorkerData> bwData,
                             bool selfReassign,
                             std::vector<Future<Void>>& toWaitOut,
                             Optional<AssignRequestType> assignType = Optional<AssignRequestType>()) {
-	// since changeBlobRange is used for assigns and revokes,
-	// we assert that assign type is specified iff this is an
 	ASSERT(active == assignType.present());
 
 	if (BW_DEBUG) {
@@ -2907,31 +2733,6 @@ static bool resumeBlobRange(Reference<BlobWorkerData> bwData, KeyRange keyRange,
 	return true;
 }
 
-ACTOR Future<Void> registerBlobWorker(Reference<BlobWorkerData> bwData, BlobWorkerInterface interf) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
-	loop {
-		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-		try {
-			Key blobWorkerListKey = blobWorkerListKeyFor(interf.id());
-			tr->addReadConflictRange(singleKeyRange(blobWorkerListKey));
-			tr->set(blobWorkerListKey, blobWorkerListValue(interf));
-
-			wait(tr->commit());
-
-			if (BW_DEBUG) {
-				fmt::print("Registered blob worker {}\n", interf.id().toString());
-			}
-			return Void();
-		} catch (Error& e) {
-			if (BW_DEBUG) {
-				fmt::print("Registering blob worker {0} got error {1}\n", interf.id().toString(), e.name());
-			}
-			wait(tr->onError(e));
-		}
-	}
-}
-
 // the contract of handleRangeAssign and handleRangeRevoke is that they change the mapping before doing any waiting.
 // This ensures GetGranuleAssignment returns an up-to-date set of ranges
 ACTOR Future<Void> handleRangeAssign(Reference<BlobWorkerData> bwData,
@@ -2968,6 +2769,9 @@ ACTOR Future<Void> handleRangeAssign(Reference<BlobWorkerData> bwData,
 		}
 		return Void();
 	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled) {
+			throw e;
+		}
 		if (BW_DEBUG) {
 			fmt::print("AssignRange [{0} - {1}) ({2}, {3}) in BW {4} got error {5}\n",
 			           req.keyRange.begin.printable().c_str(),
@@ -2987,11 +2791,21 @@ ACTOR Future<Void> handleRangeAssign(Reference<BlobWorkerData> bwData,
 
 			if (canReplyWith(e)) {
 				req.reply.sendError(e);
-				// TODO: should we just return here rather than throw and kill BW
 			}
 		}
 
-		throw;
+		TraceEvent(SevError, "BlobWorkerUnexpectedErrorRangeAssign", bwData->id)
+		    .error(e)
+		    .detail("Range", req.keyRange)
+		    .detail("ManagerEpoch", req.managerEpoch)
+		    .detail("SeqNo", req.managerSeqno);
+		ASSERT_WE_THINK(false);
+
+		// if not simulation, kill the BW
+		if (bwData->fatalError.canBeSet()) {
+			bwData->fatalError.sendError(e);
+		}
+		throw e;
 	}
 }
 
@@ -3019,6 +2833,32 @@ ACTOR Future<Void> handleRangeRevoke(Reference<BlobWorkerData> bwData, RevokeBlo
 	}
 }
 
+ACTOR Future<Void> registerBlobWorker(Reference<BlobWorkerData> bwData, BlobWorkerInterface interf) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
+	loop {
+		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		try {
+			Key blobWorkerListKey = blobWorkerListKeyFor(interf.id());
+			// FIXME: should be able to remove this conflict range
+			tr->addReadConflictRange(singleKeyRange(blobWorkerListKey));
+			tr->set(blobWorkerListKey, blobWorkerListValue(interf));
+
+			wait(tr->commit());
+
+			if (BW_DEBUG) {
+				fmt::print("Registered blob worker {}\n", interf.id().toString());
+			}
+			return Void();
+		} catch (Error& e) {
+			if (BW_DEBUG) {
+				fmt::print("Registering blob worker {0} got error {1}\n", interf.id().toString(), e.name());
+			}
+			wait(tr->onError(e));
+		}
+	}
+}
+
 ACTOR Future<Void> monitorRemoval(Reference<BlobWorkerData> bwData) {
 	state Key blobWorkerListKey = blobWorkerListKeyFor(bwData->id);
 	loop {
@@ -3030,6 +2870,7 @@ ACTOR Future<Void> monitorRemoval(Reference<BlobWorkerData> bwData) {
 
 				Optional<Value> val = wait(tr.get(blobWorkerListKey));
 				if (!val.present()) {
+					TEST(true); // Blob worker found out BM killed it from reading DB
 					return Void();
 				}
 
@@ -3054,21 +2895,17 @@ ACTOR Future<Void> runGRVChecks(Reference<BlobWorkerData> bwData) {
 	loop {
 		// only do grvs to get committed version if we need it to persist delta files
 		while (bwData->grvVersion.numWaiting() == 0) {
-			// printf("GRV checker sleeping\n");
 			wait(bwData->doGRVCheck.getFuture());
 			bwData->doGRVCheck = Promise<Void>();
-			// printf("GRV checker waking: %d pending\n", bwData->grvVersion.numWaiting());
 		}
 
 		// batch potentially multiple delta files into one GRV, and also rate limit GRVs for this worker
-		wait(delay(0.1)); // TODO KNOB?
-		// printf("GRV checker doing grv @ %.2f\n", now());
+		wait(delay(SERVER_KNOBS->BLOB_WORKER_BATCH_GRV_INTERVAL));
 
 		tr.reset();
 		try {
 			Version readVersion = wait(tr.getReadVersion());
 			ASSERT(readVersion >= bwData->grvVersion.get());
-			// printf("GRV checker got GRV %lld\n", readVersion);
 			bwData->grvVersion.set(readVersion);
 
 			++bwData->stats.commitVersionChecks;
@@ -3257,6 +3094,10 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 				TraceEvent("BlobWorkerRemoved", self->id);
 				break;
 			}
+			when(wait(self->fatalError.getFuture())) {
+				TraceEvent(SevError, "BlobWorkerActorCollectionFatalErrorNotError", self->id);
+				ASSERT(false);
+			}
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_operation_cancelled) {

From 4b254d259ceaa02feea2befa8e0e22deaf509b1d Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 11:54:57 -0600
Subject: [PATCH 188/210] Ensuring BM split retry is idempotent

---
 fdbserver/BlobManager.actor.cpp       | 82 ++++++++++++++++++++++-----
 fdbserver/BlobWorker.actor.cpp        |  6 +-
 fdbserver/ClusterController.actor.cpp |  1 -
 3 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index c914b13ae7..a20a6a40eb 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -883,24 +883,15 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		coalescedRanges.push_back(coalescedRanges.arena(), newRanges.back());
 		ASSERT(coalescedRanges.size() == SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 		if (BM_DEBUG) {
-			fmt::print(
-			    "Downsampled split from {0} -> {1} granules", newRanges.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT);
+			fmt::print("Downsampled split from {0} -> {1} granules\n",
+			           newRanges.size() - 1,
+			           SERVER_KNOBS->BG_MAX_SPLIT_FANOUT);
 		}
 
 		newRanges = coalescedRanges;
 		ASSERT(newRanges.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3}:\n",
-		           granuleRange.begin.printable(),
-		           granuleRange.end.printable(),
-		           newRanges.size() - 1,
-		           latestVersion);
-		for (int i = 0; i < newRanges.size(); i++) {
-			fmt::print("    {}\n", newRanges[i].printable());
-		}
-	}
 	ASSERT(granuleRange.begin == newRanges.front());
 	ASSERT(granuleRange.end == newRanges.back());
 
@@ -913,6 +904,19 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		newGranuleIDs.push_back(deterministicRandom()->randomUniqueID());
 	}
 
+	if (BM_DEBUG) {
+		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3}:\n",
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           newRanges.size() - 1,
+		           latestVersion);
+		for (int i = 0; i < newRanges.size(); i++) {
+			fmt::print("    {}:{}\n",
+			           (i < newGranuleIDs.size() ? newGranuleIDs[i] : UID()).toString().substr(0, 6).c_str(),
+			           newRanges[i].printable());
+		}
+	}
+
 	// Need to split range. Persist intent to split and split metadata to DB BEFORE sending split assignments to blob
 	// workers, so that nothing is lost on blob manager recovery
 	loop {
@@ -925,6 +929,47 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			// make sure we're still manager when this transaction gets committed
 			wait(checkManagerLock(tr, bmData));
 
+			// TODO can do this + lock in parallel
+			// Read splitState to see if anything was committed instead of reading granule mapping because we don't want
+			// to conflict with mapping changes/reassignments
+			state RangeResult existingState =
+			    wait(tr->getRange(blobGranuleSplitKeyRangeFor(granuleID), SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2));
+			ASSERT_WE_THINK(!existingState.more && existingState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
+			// maybe someone decreased the knob, we should gracefully handle it not in simulation
+			if (existingState.more || existingState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT) {
+				RangeResult tryAgain = wait(tr->getRange(blobGranuleSplitKeyRangeFor(granuleID), 10000));
+				ASSERT(!tryAgain.more);
+				existingState = tryAgain;
+			}
+			if (!existingState.empty()) {
+				// Something was previously committed, we must go with that decision.
+				// Read its boundaries and override our planned split boundaries
+				TEST(true); // Overriding split ranges with existing ones from DB
+				RangeResult existingBoundaries = wait(tr->getRange(
+				    KeyRangeRef(granuleRange.begin.withPrefix(blobGranuleMappingKeys.begin),
+				                keyAfter(granuleRange.end).withPrefix(blobGranuleMappingKeys.begin)),
+				    existingState.size() + 1)); // +1 because this is boundaries and existingState was granules
+				ASSERT(!existingBoundaries.more);
+				ASSERT(existingBoundaries.size() == existingState.size() + 1);
+				newRanges.clear();
+				newRanges.arena().dependsOn(existingBoundaries.arena());
+				for (auto& it : existingBoundaries) {
+					newRanges.push_back(newRanges.arena(), it.key.removePrefix(blobGranuleMappingKeys.begin));
+				}
+				ASSERT(newRanges.front() == granuleRange.begin);
+				ASSERT(newRanges.back() == granuleRange.end);
+				if (BM_DEBUG) {
+					fmt::print("Replaced old range splits for [{0} - {1}) with {2}:\n",
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable(),
+					           newRanges.size() - 1);
+					for (int i = 0; i < newRanges.size(); i++) {
+						fmt::print("    {}\n", newRanges[i].printable());
+					}
+				}
+				break;
+			}
+
 			// acquire lock for old granule to make sure nobody else modifies it
 			state Key lockKey = blobGranuleLockKeyFor(granuleRange);
 			Optional<Value> lockValue = wait(tr->get(lockKey));
@@ -955,8 +1000,8 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				if (!(bmData->epoch > ownerEpoch || (bmData->epoch == ownerEpoch && newLockSeqno > ownerSeqno))) {
 					fmt::print("BM seqno for granule [{0} - {1}) out of order for lock! manager: ({2}, {3}), owner: "
 					           "({4}, {5}})\n",
-					           granuleRange.begin.printable().c_str(),
-					           granuleRange.end.printable().c_str(),
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable(),
 					           bmData->epoch,
 					           newLockSeqno,
 					           ownerEpoch,
@@ -966,6 +1011,11 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			} else if (bmData->epoch == ownerEpoch && newLockSeqno < ownerSeqno) {
 				// we retried, and between retries we reassigned this range elsewhere. Cancel this split
 				TEST(true); // BM maybe split cancelled by subsequent move
+				if (BM_DEBUG) {
+					fmt::print("Splitting range [{0} - {1}) cancelled by move elsewhere!\n",
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable());
+				}
 				return Void();
 			}
 
@@ -1036,6 +1086,10 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		bmData->rangesToAssign.send(raAssignSplit);
 	}
 
+	// ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
+	// before returning
+	wait(bmData->rangesToAssign.onEmpty());
+
 	return Void();
 }
 
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 4ec0c18886..97e86fdc4d 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -329,12 +329,12 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
                                            BlobGranuleSplitState newState) {
 	state KeyRange currentRange = blobGranuleSplitKeyRangeFor(parentGranuleID);
 
-	state RangeResult totalState = wait(tr->getRange(currentRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2));
+	state RangeResult totalState = wait(tr->getRange(currentRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1));
 	// FIXME: remove above conflict range?
 	tr->addWriteConflictRange(currentRange);
-	ASSERT_WE_THINK(!totalState.more && totalState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
+	ASSERT_WE_THINK(!totalState.more && totalState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT);
 	// maybe someone decreased the knob, we should gracefully handle it not in simulation
-	if (totalState.more || totalState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1) {
+	if (totalState.more || totalState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT) {
 		RangeResult tryAgain = wait(tr->getRange(currentRange, 10000));
 		ASSERT(!tryAgain.more);
 		totalState = tryAgain;
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 36a0cd3228..899522f229 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2125,7 +2125,6 @@ ACTOR Future<int64_t> getNextBMEpoch(ClusterControllerData* self) {
 			wait(tr->commit());
 			return newEpoch;
 		} catch (Error& e) {
-			printf("Acquiring blob manager lock got error %s\n", e.name());
 			wait(tr->onError(e));
 		}
 	}

From d7367dad99b965e1b6f1a370bd4157cb41dc93e5 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 11:56:22 -0600
Subject: [PATCH 189/210] Simplifying CF gap validation to ignore SS CF fetches

---
 fdbserver/storageserver.actor.cpp                 | 7 +++++++
 fdbserver/workloads/BlobGranuleVerifier.actor.cpp | 7 ++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index b35fe3f611..483283d470 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2032,6 +2032,13 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			// gap validation
 			while (memoryVerifyIdx < memoryReply.mutations.size() &&
 			       version > memoryReply.mutations[memoryVerifyIdx].version) {
+				if (req.canReadPopped) {
+					// There are weird cases where SS fetching mixed with SS durability and popping can mean there are
+					// gaps before the popped version temporarily
+					memoryVerifyIdx++;
+					continue;
+				}
+
 				// There is a case where this can happen - if we wait on a fetching change feed, and the feed is
 				// popped while we wait, we could have copied the memory mutations into memoryReply before the
 				// pop, but they may or may not have been skipped writing to disk
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index 04aa5e936d..c8f69f3c6f 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -534,9 +534,10 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		clients.push_back(timeout(findGranules(cx, this), testDuration, Void()));
 		for (int i = 0; i < threads; i++) {
 			clients.push_back(
-			    timeout(reportErrors(verifyGranules(
-			                             cx, this, clientId == 0 && i == 0 && deterministicRandom()->random01() < 0.5),
-			                         "BlobGranuleVerifier"),
+			    timeout(reportErrors(
+			                verifyGranules(
+			                    cx, this, false /*clientId == 0 && i == 0 && deterministicRandom()->random01() < 0.5*/),
+			                "BlobGranuleVerifier"),
 			            testDuration,
 			            Void()));
 		}

From ba35e7908a29ab5d1ba45cefab22ea779a76a0d5 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 12:11:44 -0600
Subject: [PATCH 190/210] off by one

---
 fdbserver/BlobManager.actor.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index a20a6a40eb..abb16038e5 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -945,10 +945,11 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				// Something was previously committed, we must go with that decision.
 				// Read its boundaries and override our planned split boundaries
 				TEST(true); // Overriding split ranges with existing ones from DB
-				RangeResult existingBoundaries = wait(tr->getRange(
-				    KeyRangeRef(granuleRange.begin.withPrefix(blobGranuleMappingKeys.begin),
-				                keyAfter(granuleRange.end).withPrefix(blobGranuleMappingKeys.begin)),
-				    existingState.size() + 1)); // +1 because this is boundaries and existingState was granules
+				RangeResult existingBoundaries =
+				    wait(tr->getRange(KeyRangeRef(granuleRange.begin.withPrefix(blobGranuleMappingKeys.begin),
+				                                  keyAfter(granuleRange.end).withPrefix(blobGranuleMappingKeys.begin)),
+				                      existingState.size() + 2));
+				// +2 because this is boundaries and existingState was granules, and to ensure it doesn't set more
 				ASSERT(!existingBoundaries.more);
 				ASSERT(existingBoundaries.size() == existingState.size() + 1);
 				newRanges.clear();

From c35e020da73d4045ad0f432fc697f54b702c8810 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 12:38:23 -0600
Subject: [PATCH 191/210] SS and BM test macros

---
 fdbserver/BlobManager.actor.cpp               | 19 +++++++++++++++++--
 fdbserver/storageserver.actor.cpp             |  9 +++++++++
 .../workloads/BlobGranuleVerifier.actor.cpp   |  1 +
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index abb16038e5..7c684dacac 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -270,6 +270,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 			           estimated.bytes);
 		}
 
+		TEST(writeHot); // Change feed write hot split
 		if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES || writeHot) {
 			// only split on bytes and write rate
 			state StorageMetrics splitMetrics;
@@ -335,6 +336,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
 	// wait until there are BWs to pick from
 	while (bmData->workerStats.size() == 0) {
+		TEST(true); // BM wants to assign range, but no workers available
 		if (BM_DEBUG) {
 			fmt::print("BM {0} waiting for blob workers before assigning granules\n", bmData->epoch);
 		}
@@ -460,6 +462,8 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			throw;
 		}
 
+		TEST(true); // BM retrying range assign
+
 		// We use reliable delivery (getReply), so the broken_promise means the worker is dead, and we may need to retry
 		// somewhere else
 		if (assignment.isAssign) {
@@ -533,6 +537,7 @@ ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 				if (assignment.assign.get().type == AssignRequestType::Continue) {
 					ASSERT(assignment.worker.present());
 					if (i.range() != assignment.keyRange || i.cvalue() != assignment.worker.get()) {
+						TEST(true); // BM assignment out of date
 						if (BM_DEBUG) {
 							fmt::print("Out of date re-assign for ({0}, {1}). Assignment must have changed while "
 							           "checking split.\n  Reassign: [{2} - {3}): {4}\n  Existing: [{5} - {6}): {7}\n",
@@ -1087,8 +1092,8 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		bmData->rangesToAssign.send(raAssignSplit);
 	}
 
-	// ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
-	// before returning
+	// Ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
+	// before returning. This prevents a race with a subsequent split evaluation
 	wait(bmData->rangesToAssign.onEmpty());
 
 	return Void();
@@ -1350,6 +1355,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 			// if it is permanent, the failure monitor will eventually trip.
 			ASSERT(e.code() != error_code_end_of_stream);
 			if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_connection_failed) {
+				TEST(true); // BM retrying BW monitoring
 				wait(delay(backoff));
 				backoff = std::min(backoff * SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,
 				                   SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX);
@@ -1503,6 +1509,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 					std::get<0>(old.value()) = UID();
 				}
 				if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) {
+					TEST(true); // BM Recovery: DB disagrees with workers
 					outOfDate.push_back(std::pair(oldWorker, old.range()));
 				}
 			} else if (oldWorker != UID() && oldWorker != newId &&
@@ -1510,6 +1517,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 				// 2 blob workers reported conflicting mappings, add old one to out of date (if not already added by a
 				// previous intersecting range in the split case)
 				if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) {
+					TEST(true); // BM Recovery: Two workers claim ownership of same granule
 					outOfDate.push_back(std::pair(oldWorker, old.range()));
 				}
 			}
@@ -1555,6 +1563,8 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		return Void();
 	}
 
+	TEST(true); // BM doing recovery
+
 	wait(delay(0));
 
 	// At this point, bmData->workersById is a list of all alive blob workers, but could also include some dead BWs.
@@ -1634,6 +1644,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 				bmData->workerStats[workerId].numGranulesAssigned = reply.get().assignments.size();
 			}
 		} else {
+			TEST(true); // BM Recovery: BW didn't respond to assignments request
 			// SOMEDAY: mark as failed and kill it
 			if (BM_DEBUG) {
 				fmt::print("  Worker {}: failed\n", workerId.toString().substr(0, 5));
@@ -1804,6 +1815,7 @@ ACTOR Future<Void> chaosRangeMover(Reference<BlobManagerData> bmData) {
 	// KeyRange isn't hashable and this is only for simulation, so just use toString of range
 	state std::unordered_set<std::string> alreadyMoved;
 	ASSERT(g_network->isSimulated());
+	TEST(true); // BM chaos range mover enabled
 	loop {
 		wait(delay(30.0));
 
@@ -1906,6 +1918,7 @@ ACTOR Future<Void> initializeBlobWorker(Reference<BlobManagerData> self, Recruit
 		// if it failed in an expected way, add some delay before we try to recruit again
 		// on this worker
 		if (newBlobWorker.isError()) {
+			TEST(true); // BM got error recruiting BW
 			TraceEvent(SevWarn, "BMRecruitmentError").error(newBlobWorker.getError());
 			if (!newBlobWorker.isError(error_code_recruitment_failed) &&
 			    !newBlobWorker.isError(error_code_request_maybe_delivered)) {
@@ -2561,6 +2574,8 @@ ACTOR Future<Void> doLockChecks(Reference<BlobManagerData> bmData) {
 		wait(check.getFuture());
 		wait(delay(0.5)); // don't do this too often if a lot of conflict
 
+		TEST(true); // BM doing lock checks after getting conflicts
+
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 
 		loop {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 483283d470..8862544c10 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2118,9 +2118,11 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			// If still empty, that means disk results were filtered out, but skipped all memory results. Add an empty,
 			// either the last version from disk
 			if (reply.mutations.empty() && res.size()) {
+				TEST(true); // Change feed adding empty version after disk + memory filtered
 				reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 			}
 		} else if (reply.mutations.empty() || reply.mutations.back().version < lastVersion) {
+			TEST(true); // Change feed adding empty version after disk filtered
 			reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 		}
 	} else {
@@ -2131,6 +2133,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	Version finalVersion = std::min(req.end - 1, dequeVersion);
 	if ((reply.mutations.empty() || reply.mutations.back().version < finalVersion) && remainingLimitBytes > 0 &&
 	    remainingDurableBytes > 0) {
+		TEST(true); // Change feed adding empty version after empty results
 		reply.mutations.push_back(
 		    reply.arena, MutationsAndVersionRef(finalVersion, finalVersion == dequeVersion ? dequeKnownCommit : 0));
 		// if we add empty mutation after the last thing in memory, and didn't read from disk, gotAll is true
@@ -2305,6 +2308,7 @@ ACTOR Future<Void> stopChangeFeedOnMove(StorageServer* data, ChangeFeedStreamReq
 		}
 		return Void();
 	}
+	TEST(true); // Change feed moved away cancelling queries
 	// DO NOT call req.reply.onReady before sending - we need to propagate this error through regardless of how far
 	// behind client is
 	req.reply.sendError(wrong_shard_server());
@@ -4759,6 +4763,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 						lastVersion = remoteVersion;
 						versionsFetched++;
 					} else {
+						TEST(true); // Change feed ignoring write on move because it was popped concurrently
 						if (MUTATION_TRACKING_ENABLED) {
 							for (auto& m : remoteResult[remoteLoc].mutations) {
 								DEBUG_MUTATION("ChangeFeedWriteMoveIgnore", remoteVersion, m, data->thisServerID)
@@ -4827,6 +4832,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 
 	// if we were popped or removed while fetching but it didn't pass the fetch version while writing, clean up here
 	if (versionsFetched > 0 && startVersion < changeFeedInfo->emptyVersion) {
+		TEST(true); // Change feed cleaning up popped data after move
 		ASSERT(firstVersion != invalidVersion);
 		ASSERT(lastVersion != invalidVersion);
 		Version endClear = std::min(lastVersion + 1, changeFeedInfo->emptyVersion);
@@ -4872,6 +4878,7 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 
 	auto cleanupPending = data->changeFeedCleanupDurable.find(changeFeedInfo->id);
 	if (cleanupPending != data->changeFeedCleanupDurable.end()) {
+		TEST(true); // Change feed waiting for dirty previous move to finish
 		TraceEvent(SevDebug, "FetchChangeFeedWaitCleanup", data->thisServerID)
 		    .detail("RangeID", changeFeedInfo->id.printable())
 		    .detail("Range", changeFeedInfo->range.toString())
@@ -4964,6 +4971,7 @@ ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data, KeyR
 			auto feedCleanup = data->changeFeedCleanupDurable.find(cfEntry.rangeId);
 
 			if (cfEntry.stopVersion < changeFeedInfo->stopVersion) {
+				TEST(true); // Change feed updated stop version from fetch metadata
 				changeFeedInfo->stopVersion = cfEntry.stopVersion;
 				addMutationToLog = true;
 			}
@@ -5363,6 +5371,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		std::unordered_set<Key> newChangeFeeds;
 		for (auto& r : ranges) {
 			for (auto& cfInfo : r.value()) {
+				TEST(true); // SS fetching new change feed that didn't exist when fetch started
 				newChangeFeeds.insert(cfInfo->id);
 			}
 		}
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index c8f69f3c6f..09d19c6db5 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -535,6 +535,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		for (int i = 0; i < threads; i++) {
 			clients.push_back(
 			    timeout(reportErrors(
+			                // TODO change back
 			                verifyGranules(
 			                    cx, this, false /*clientId == 0 && i == 0 && deterministicRandom()->random01() < 0.5*/),
 			                "BlobGranuleVerifier"),

From 903f7adbc4006d557d8e331dab43301106df99cf Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 13:53:06 -0600
Subject: [PATCH 192/210] more cleanup

---
 fdbclient/NativeAPI.actor.cpp               |  1 +
 fdbserver/BlobGranuleServerCommon.actor.cpp | 17 +++------------
 fdbserver/BlobGranuleServerCommon.actor.h   |  9 ++------
 fdbserver/BlobManager.actor.cpp             | 14 ++++++-------
 fdbserver/BlobWorker.actor.cpp              | 23 ++++++++++++++++-----
 fdbserver/storageserver.actor.cpp           |  5 ++++-
 6 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index bb0d0143fc..a747ecf77c 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6747,6 +6747,7 @@ Future<Standalone<VectorRef<KeyRef>>> Transaction::getRangeSplitPoints(KeyRange
 	return ::getRangeSplitPoints(trState, keys, chunkSize);
 }
 
+// TODO REMOVE when correctness clean
 #define BG_REQUEST_DEBUG false
 
 // the blob granule requests are a bit funky because they piggyback off the existing transaction to read from the system
diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp
index 7700d6152b..94192bb8d7 100644
--- a/fdbserver/BlobGranuleServerCommon.actor.cpp
+++ b/fdbserver/BlobGranuleServerCommon.actor.cpp
@@ -45,12 +45,7 @@ ACTOR Future<Optional<GranuleHistory>> getLatestGranuleHistory(Transaction* tr,
 
 // Gets the files based on the file key range [startKey, endKey)
 // and populates the files object accordingly
-ACTOR Future<Void> readGranuleFiles(Transaction* tr,
-                                    Key* startKey,
-                                    Key endKey,
-                                    GranuleFiles* files,
-                                    UID granuleID,
-                                    bool debug) {
+ACTOR Future<Void> readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, GranuleFiles* files, UID granuleID) {
 
 	loop {
 		int lim = BUGGIFY ? 2 : 1000;
@@ -85,18 +80,12 @@ ACTOR Future<Void> readGranuleFiles(Transaction* tr,
 			break;
 		}
 	}
-	if (debug) {
-		printf("Loaded %d snapshot and %d delta files for %s\n",
-		       files->snapshotFiles.size(),
-		       files->deltaFiles.size(),
-		       granuleID.toString().c_str());
-	}
 	return Void();
 }
 
 // Wrapper around readGranuleFiles
 // Gets all files belonging to the granule with id granule ID
-ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID, bool debug) {
+ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID) {
 	state KeyRange range = blobGranuleFileKeyRangeFor(granuleID);
 	state Key startKey = range.begin;
 	state GranuleFiles files;
@@ -106,7 +95,7 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID, bool deb
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID, debug));
+			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
 			wait(tr.onError(e));
diff --git a/fdbserver/BlobGranuleServerCommon.actor.h b/fdbserver/BlobGranuleServerCommon.actor.h
index b593cc1361..d48418c951 100644
--- a/fdbserver/BlobGranuleServerCommon.actor.h
+++ b/fdbserver/BlobGranuleServerCommon.actor.h
@@ -64,12 +64,7 @@ struct GranuleFiles {
 
 class Transaction;
 ACTOR Future<Optional<GranuleHistory>> getLatestGranuleHistory(Transaction* tr, KeyRange range);
-ACTOR Future<Void> readGranuleFiles(Transaction* tr,
-                                    Key* startKey,
-                                    Key endKey,
-                                    GranuleFiles* files,
-                                    UID granuleID,
-                                    bool debug);
+ACTOR Future<Void> readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, GranuleFiles* files, UID granuleID);
 
-ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID, bool debug);
+ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID);
 #endif
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 7c684dacac..c8d8a65ea8 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -2043,7 +2043,7 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobManagerData> bmData, U
 	state GranuleFiles files;
 	loop {
 		try {
-			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID, BM_DEBUG));
+			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -2062,7 +2062,7 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self, UID granu
 	}
 
 	// get files
-	GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId, BM_DEBUG));
+	GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId));
 
 	std::vector<Future<Void>> deletions;
 	std::vector<std::string> filesToDelete; // TODO: remove, just for debugging
@@ -2134,7 +2134,7 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self, UID g
 	}
 
 	// get files
-	GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId, BM_DEBUG));
+	GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId));
 
 	// represents the version of the latest snapshot file in this granule with G.version < pruneVersion
 	Version latestSnapshotVersion = invalidVersion;
@@ -2605,8 +2605,8 @@ ACTOR Future<Void> doLockChecks(Reference<BlobManagerData> bmData) {
 	}
 }
 
-ACTOR Future<Void> blobManagerExclusionSafetyCheck(Reference<BlobManagerData> self,
-                                                   BlobManagerExclusionSafetyCheckRequest req) {
+static void blobManagerExclusionSafetyCheck(Reference<BlobManagerData> self,
+                                            BlobManagerExclusionSafetyCheckRequest req) {
 	TraceEvent("BMExclusionSafetyCheckBegin", self->id).log();
 	BlobManagerExclusionSafetyCheckReply reply(true);
 	// make sure at least one blob worker remains after exclusions
@@ -2632,8 +2632,6 @@ ACTOR Future<Void> blobManagerExclusionSafetyCheck(Reference<BlobManagerData> se
 
 	TraceEvent("BMExclusionSafetyCheckEnd", self->id).log();
 	req.reply.send(reply);
-
-	return Void();
 }
 
 ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
@@ -2692,7 +2690,7 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 			}
 			when(BlobManagerExclusionSafetyCheckRequest exclCheckReq =
 			         waitNext(bmInterf.blobManagerExclCheckReq.getFuture())) {
-				self->addActor.send(blobManagerExclusionSafetyCheck(self, exclCheckReq));
+				blobManagerExclusionSafetyCheck(self, exclCheckReq);
 			}
 			when(wait(collection)) {
 				TraceEvent("BlobManagerActorCollectionError");
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 97e86fdc4d..46ad75f78b 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -281,7 +281,7 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobWorkerData> bwData, UI
 	state GranuleFiles files;
 	loop {
 		try {
-			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID, BW_DEBUG));
+			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -296,7 +296,7 @@ ACTOR Future<GranuleFiles> loadPreviousFiles(Transaction* tr, UID granuleID) {
 	// no need to add conflict range for read b/c of granule lock
 	state Key startKey = range.begin;
 	state GranuleFiles files;
-	wait(readGranuleFiles(tr, &startKey, range.end, &files, granuleID, BW_DEBUG));
+	wait(readGranuleFiles(tr, &startKey, range.end, &files, granuleID));
 	return files;
 }
 
@@ -967,7 +967,9 @@ ACTOR Future<Void> handleCompletedDeltaFile(Reference<BlobWorkerData> bwData,
 
 	if (completedDeltaFile.version > cfStartVersion) {
 		if (BW_DEBUG) {
-			fmt::print("Popping change feed {0} at {1}\n", cfKey.printable(), completedDeltaFile.version);
+			fmt::print("Popping change feed {0} at {1}\n",
+			           cfKeyToGranuleID(cfKey).toString().c_str(),
+			           completedDeltaFile.version);
 		}
 		// FIXME: for a write-hot shard, we could potentially batch these and only pop the largest one after several
 		// have completed
@@ -1595,7 +1597,8 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 
 								DEBUG_MUTATION("BlobWorkerBuffer", deltas.version, delta, bwData->id)
 								    .detail("Granule", metadata->keyRange)
-								    .detail("ChangeFeedID", readOldChangeFeed ? oldCFKey.get() : cfKey)
+								    .detail("ChangeFeedID",
+								            cfKeyToGranuleID(readOldChangeFeed ? oldCFKey.get() : cfKey))
 								    .detail("OldChangeFeed", readOldChangeFeed ? "T" : "F");
 							}
 							metadata->currentDeltas.push_back_deep(metadata->currentDeltas.arena(), deltas);
@@ -1990,6 +1993,16 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// if we don't have to wait for change feed version to catch up or wait for any pending file writes to complete,
 	// nothing to do
 
+	if (BW_REQUEST_DEBUG) {
+		printf("WFV %lld) CF=%lld, pendingD=%lld, durableD=%lld, pendingS=%lld, durableS=%lld\n",
+		       v,
+		       metadata->activeCFData.get()->getVersion(),
+		       metadata->pendingDeltaVersion,
+		       metadata->durableDeltaVersion.get(),
+		       metadata->pendingSnapshotVersion,
+		       metadata->durableSnapshotVersion.get());
+	}
+
 	ASSERT(metadata->activeCFData.get().isValid());
 
 	if (v <= metadata->activeCFData.get()->getVersion() &&
@@ -2163,7 +2176,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				// lazily load files for old granule if not present
 				chunkRange = cur->range;
 				if (!cur->files.isValid() || cur->files.isError()) {
-					cur->files = loadHistoryFiles(bwData->db, cur->granuleID, BW_DEBUG);
+					cur->files = loadHistoryFiles(bwData->db, cur->granuleID);
 				}
 
 				choose {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 8862544c10..7b08e6548d 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -1204,7 +1204,10 @@ public:
 		auto& clientVersions = changeFeedClientVersions[addr];
 		Version minVersion = version.get();
 		for (auto& it : clientVersions) {
-			// fmt::print("Blocked client {0} @ {1}\n", it.first.toString().substr(0, 8), it.second);
+			/*fmt::print("SS {0} Blocked client {1} @ {2}\n",
+			           thisServerID.toString().substr(0, 4),
+			           it.first.toString().substr(0, 8),
+			           it.second);*/
 			minVersion = std::min(minVersion, it.second);
 		}
 		return minVersion;

From 211e5b579a57eaef67a8992623f667f151253226 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 13:55:42 -0600
Subject: [PATCH 193/210] Adding back debug stream uid to help track down bugs

---
 fdbclient/NativeAPI.actor.cpp      | 5 +++++
 fdbclient/StorageServerInterface.h | 4 +++-
 fdbserver/storageserver.actor.cpp  | 4 +++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a747ecf77c..4be06507e8 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7788,6 +7788,9 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		if (replyBufferSize != -1 && req.replyBufferSize < CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES) {
 			req.replyBufferSize = CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES;
 		}
+		// TODO REMOVE
+		req.debugUID = deterministicRandom()->randomUniqueID();
+
 		results->streams.push_back(it.first.changeFeedStream.getReplyStream(req));
 	}
 
@@ -7934,6 +7937,8 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 	req.range = range;
 	req.canReadPopped = canReadPopped;
 	req.replyBufferSize = replyBufferSize;
+	// TODO REMOVE
+	req.debugUID = deterministicRandom()->randomUniqueID();
 
 	results->streams.clear();
 
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index cbe6937f2f..794397cfb2 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -793,13 +793,15 @@ struct ChangeFeedStreamRequest {
 	KeyRange range;
 	int replyBufferSize = -1;
 	bool canReadPopped = true;
+	// TODO REMOVE
+	UID debugUID;
 
 	ReplyPromiseStream<ChangeFeedStreamReply> reply;
 
 	ChangeFeedStreamRequest() {}
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeID, begin, end, range, reply, spanContext, replyBufferSize, canReadPopped, arena);
+		serializer(ar, rangeID, begin, end, range, reply, spanContext, replyBufferSize, canReadPopped, debugUID, arena);
 	}
 };
 
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 7b08e6548d..86fba6190b 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -7884,7 +7884,9 @@ ACTOR Future<Void> serveChangeFeedStreamRequests(StorageServer* self,
 		ChangeFeedStreamRequest req = waitNext(changeFeedStream);
 		// must notify change feed that its shard is moved away ASAP
 
-		UID streamUID = deterministicRandom()->randomUniqueID();
+		// TODO CHANGE BACK
+		// UID streamUID = deterministicRandom()->randomUniqueID();
+		UID streamUID = req.debugUID;
 		self->actors.add(changeFeedStreamQ(self, req, streamUID) || stopChangeFeedOnMove(self, req, streamUID));
 	}
 }

From 4f88e0040408667b2a4eb1c5c057492dbb83207e Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 14:37:32 -0600
Subject: [PATCH 194/210] Adding change feed pop to set of expected granule
 errors

---
 fdbserver/BlobWorker.actor.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 46ad75f78b..fccb73b5a8 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -1810,7 +1810,15 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		}
 
 		if (e.code() == error_code_granule_assignment_conflict) {
-			TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id).detail("Granule", metadata->keyRange);
+			TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id)
+			    .detail("Granule", metadata->keyRange)
+			    .detail("GranuleID", startState.granuleID);
+			return Void();
+		}
+		if (e.code() == error_code_change_feed_popped) {
+			TraceEvent(SevInfo, "GranuleGotChangeFeedPopped", bwData->id)
+			    .detail("Granule", metadata->keyRange)
+			    .detail("GranuleID", startState.granuleID);
 			return Void();
 		}
 		++bwData->stats.granuleUpdateErrors;

From edb65f4aa9b1a9672fb2bfefbdecbe885c694920 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 16:33:41 -0600
Subject: [PATCH 195/210] Exclude redwood from BGVerifySmall because it can
 make a root page too large with massive KV pairs in change feeds

---
 tests/fast/BlobGranuleVerifySmall.toml      | 1 +
 tests/fast/BlobGranuleVerifySmallClean.toml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/fast/BlobGranuleVerifySmall.toml b/tests/fast/BlobGranuleVerifySmall.toml
index b3852c428f..22a4b15ae6 100644
--- a/tests/fast/BlobGranuleVerifySmall.toml
+++ b/tests/fast/BlobGranuleVerifySmall.toml
@@ -1,5 +1,6 @@
 [configuration]
 blobGranulesEnabled = true 
+storageEngineExcludeTypes = [3] # FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet
 
 [[test]]
 testTitle = 'BlobGranuleVerifySmall'
diff --git a/tests/fast/BlobGranuleVerifySmallClean.toml b/tests/fast/BlobGranuleVerifySmallClean.toml
index 9378ead90c..0a7d2a95d6 100644
--- a/tests/fast/BlobGranuleVerifySmallClean.toml
+++ b/tests/fast/BlobGranuleVerifySmallClean.toml
@@ -1,5 +1,6 @@
 [configuration]
 blobGranulesEnabled = true
+storageEngineExcludeTypes = [3] # FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet
 
 [[test]]
 testTitle = 'BlobGranuleVerifySmallClean'

From 479ac313ca3a19469d5f40b16e5e175cfb32f25f Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 16:19:57 -0600
Subject: [PATCH 196/210] Fixing conflict handling in BM recovery

---
 fdbserver/BlobManager.actor.cpp | 11 ++++++++---
 fdbserver/BlobWorker.actor.cpp  | 16 +++++++++-------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index c8d8a65ea8..8d1b839e38 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -1476,7 +1476,6 @@ ACTOR Future<Void> checkBlobWorkerList(Reference<BlobManagerData> bmData, Promis
 // Resolves these conflicts by comparing the epoch + seqno for the range
 // Special epoch/seqnos:
 //   (0,0): range is not mapped
-//   (0,1): range is mapped, but worker is unknown
 static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
                           const KeyRangeRef& newRange,
                           UID newId,
@@ -1493,6 +1492,10 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 		int64_t oldSeqno = std::get<2>(old.value());
 		if (oldEpoch > newEpoch || (oldEpoch == newEpoch && oldSeqno > newSeqno)) {
 			newer.push_back(std::pair(old.range(), std::tuple(oldWorker, oldEpoch, oldSeqno)));
+			if (old.range() != newRange) {
+				TEST(true); // BM Recovery: BWs disagree on range boundaries
+				anyConflicts = true;
+			}
 		} else {
 			allExistingNewer = false;
 			if (newId != UID() && newEpoch != std::numeric_limits<int64_t>::max()) {
@@ -1500,6 +1503,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 				ASSERT(oldEpoch != newEpoch || oldSeqno != newSeqno);
 			}
 			if (newEpoch == std::numeric_limits<int64_t>::max() && (oldWorker != newId || old.range() != newRange)) {
+				TEST(true); // BM Recovery: DB disagrees with workers
 				// new one is from DB (source of truth on boundaries) and existing mapping disagrees on boundary or
 				// assignment, do explicit revoke and re-assign to converge
 				anyConflicts = true;
@@ -1508,8 +1512,9 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 				if (old.range() != newRange) {
 					std::get<0>(old.value()) = UID();
 				}
-				if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) {
-					TEST(true); // BM Recovery: DB disagrees with workers
+				if (oldWorker != UID() &&
+				    (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range())))) {
+
 					outOfDate.push_back(std::pair(oldWorker, old.range()));
 				}
 			} else if (oldWorker != UID() && oldWorker != newId &&
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index fccb73b5a8..066f311163 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -979,13 +979,15 @@ ACTOR Future<Void> handleCompletedDeltaFile(Reference<BlobWorkerData> bwData,
 		wait(popFuture);
 	}
 	while (!rollbacksCompleted->empty() && completedDeltaFile.version >= rollbacksCompleted->front().second) {
-		fmt::print("Granule [{0} - {1}) on BW {2} completed rollback {3} -> {4} with delta file {5}\n",
-		           metadata->keyRange.begin.printable().c_str(),
-		           metadata->keyRange.end.printable().c_str(),
-		           bwData->id.toString().substr(0, 5).c_str(),
-		           rollbacksCompleted->front().second,
-		           rollbacksCompleted->front().first,
-		           completedDeltaFile.version);
+		if (BW_DEBUG) {
+			fmt::print("Granule [{0} - {1}) on BW {2} completed rollback {3} -> {4} with delta file {5}\n",
+			           metadata->keyRange.begin.printable().c_str(),
+			           metadata->keyRange.end.printable().c_str(),
+			           bwData->id.toString().substr(0, 5).c_str(),
+			           rollbacksCompleted->front().second,
+			           rollbacksCompleted->front().first,
+			           completedDeltaFile.version);
+		}
 		rollbacksCompleted->pop_front();
 	}
 	return Void();

From 2a506f2dff9b7274fa4d7b6d9c30fd251de3f83a Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Thu, 10 Mar 2022 20:11:37 -0600
Subject: [PATCH 197/210] Updating SS desired every mutation to avoid CF merge
 deadlock

---
 fdbclient/NativeAPI.actor.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4be06507e8..19bdaac6a2 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7608,33 +7608,33 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 						wait(results.onEmpty());
 						if (rep.mutations[resultLoc].version >= nextVersion) {
 							results.send(rep.mutations[resultLoc]);
+
+							// check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we
+							// just skip to the next wait and get actor_cancelled
+							// FIXME: this is somewhat expensive to do every mutation.
+							for (auto& it : feedData->storageData) {
+								if (refresh.canBeSet() && rep.mutations[resultLoc].version > it->desired.get()) {
+									it->desired.set(rep.mutations[resultLoc].version);
+								}
+							}
 						} else {
 							ASSERT(rep.mutations[resultLoc].mutations.empty());
 						}
 						resultLoc++;
 					}
+
 					// if we got the empty version that went backwards, don't decrease nextVersion
 					if (rep.mutations.back().version + 1 > nextVersion) {
 						nextVersion = rep.mutations.back().version + 1;
 					}
 
-					// check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we just
-					// skip to the next wait and get actor_cancelled
-
 					if (refresh.canBeSet() && !atLatestVersion && rep.atLatestVersion) {
 						atLatestVersion = true;
 						feedData->notAtLatest.set(feedData->notAtLatest.get() - 1);
 					}
-
 					if (refresh.canBeSet() && rep.minStreamVersion > storageData->version.get()) {
 						storageData->version.set(rep.minStreamVersion);
 					}
-
-					for (auto& it : feedData->storageData) {
-						if (refresh.canBeSet() && rep.mutations.back().version > it->desired.get()) {
-							it->desired.set(rep.mutations.back().version);
-						}
-					}
 				}
 				when(wait(atLatestVersion && replyStream.isEmpty() && results.isEmpty()
 				              ? storageData->version.whenAtLeast(nextVersion)

From 67dd3d071e0cd027ddd4058277d168a6e9864ec5 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Fri, 11 Mar 2022 10:49:26 -0600
Subject: [PATCH 198/210] Fixing incorrect assert in BM split handling

---
 fdbserver/BlobManager.actor.cpp | 48 ++++++++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 9 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 8d1b839e38..9a2e1cdb69 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -1074,6 +1074,14 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		}
 	}
 
+	if (BM_DEBUG) {
+		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} done, sending assignments:\n",
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           newRanges.size() - 1,
+		           latestVersion);
+	}
+
 	// transaction committed, send range assignments
 	// range could have been moved since split eval started, so just revoke from whoever has it
 	RangeAssignment raRevoke;
@@ -1092,10 +1100,27 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		bmData->rangesToAssign.send(raAssignSplit);
 	}
 
+	if (BM_DEBUG) {
+		fmt::print(
+		    "Splitting range [{0} - {1}) into {2} granules @ {3} sent assignments, waiting for them to be processed:\n",
+		    granuleRange.begin.printable(),
+		    granuleRange.end.printable(),
+		    newRanges.size() - 1,
+		    latestVersion);
+	}
+
 	// Ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
 	// before returning. This prevents a race with a subsequent split evaluation
 	wait(bmData->rangesToAssign.onEmpty());
 
+	if (BM_DEBUG) {
+		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} got assignments processed\n",
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           newRanges.size() - 1,
+		           latestVersion);
+	}
+
 	return Void();
 }
 
@@ -1268,18 +1293,21 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 				      currGranuleAssignment.end() == rep.granuleRange.end &&
 				      currGranuleAssignment.cvalue() == bwInterf.id())) {
 					if (BM_DEBUG) {
-						fmt::print(
-						    "Manager {0} ignoring status from BW {1} for granule [{2} - {3}) since BW {4} owns it.\n",
-						    bmData->epoch,
-						    bwInterf.id().toString().substr(0, 5),
-						    rep.granuleRange.begin.printable(),
-						    rep.granuleRange.end.printable(),
-						    currGranuleAssignment.cvalue().toString().substr(0, 5));
+						fmt::print("Manager {0} ignoring status from BW {1} for granule [{2} - {3}) since BW {4} owns "
+						           "[{5} - {6}).\n",
+						           bmData->epoch,
+						           bwInterf.id().toString().substr(0, 5),
+						           rep.granuleRange.begin.printable(),
+						           rep.granuleRange.end.printable(),
+						           currGranuleAssignment.cvalue().toString().substr(0, 5),
+						           currGranuleAssignment.begin().printable(),
+						           currGranuleAssignment.end().printable());
 					}
 					// FIXME: could send revoke request
 					continue;
 				}
 
+				// FIXME: We will need to go over all splits in the range once we're doing merges, instead of first one
 				auto lastSplitEval = bmData->splitEvaluations.rangeContaining(rep.granuleRange.begin);
 				if (rep.granuleRange.begin == lastSplitEval.begin() && rep.granuleRange.end == lastSplitEval.end() &&
 				    rep.epoch == lastSplitEval.cvalue().epoch && rep.seqno == lastSplitEval.cvalue().seqno) {
@@ -1298,18 +1326,20 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 						// For example, one worker asked BM to split, then died, granule was moved, new worker asks to
 						// split on recovery. We need to ensure that they are semantically the same split.
 						// We will just rely on the in-progress split to finish
-						ASSERT(lastSplitEval.range() == rep.granuleRange);
 						if (BM_DEBUG) {
 							fmt::print("Manager {0} got split request for [{1} - {2}) @ ({3}, {4}), but already in "
-							           "progress from ({5}, {6})\n",
+							           "progress from [{5} - {6}) @ ({7}, {8})\n",
 							           bmData->epoch,
 							           rep.granuleRange.begin.printable().c_str(),
 							           rep.granuleRange.end.printable().c_str(),
 							           rep.epoch,
 							           rep.seqno,
+							           lastSplitEval.begin().printable().c_str(),
+							           lastSplitEval.end().printable().c_str(),
 							           lastSplitEval.cvalue().epoch,
 							           lastSplitEval.cvalue().seqno);
 						}
+						// ignore the request, they will retry
 					} else {
 						if (BM_DEBUG) {
 							fmt::print("Manager {0} evaluating [{1} - {2}) @ ({3}, {4}) for split\n",

From 82eef0c7f7074ed2e70824b61875df6ce29682b0 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Fri, 11 Mar 2022 16:37:52 -0600
Subject: [PATCH 199/210] Better CF tracing for client and server

---
 fdbclient/NativeAPI.actor.cpp     | 39 ++++++++++++++++++++++++++++---
 fdbserver/storageserver.actor.cpp | 33 +++++++++++++++++++++++++-
 2 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 19bdaac6a2..1e3c70a733 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7570,13 +7570,16 @@ Future<Void> ChangeFeedData::whenAtLeast(Version version) {
 	return changeFeedWhenAtLatest(Reference<ChangeFeedData>::addRef(this), version);
 }
 
+#define DEBUG_CF_CLIENT_TRACE false
+
 ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
                                            PromiseStream<Standalone<MutationsAndVersionRef>> results,
                                            ReplyPromiseStream<ChangeFeedStreamReply> replyStream,
                                            Version begin,
                                            Version end,
                                            Reference<ChangeFeedData> feedData,
-                                           Reference<ChangeFeedStorageData> storageData) {
+                                           Reference<ChangeFeedStorageData> storageData,
+                                           UID debugUID /* TODO REMOVE */) {
 
 	// calling lastReturnedVersion's callbacks could cause us to be cancelled
 	state Promise<Void> refresh = feedData->refresh;
@@ -7649,6 +7652,9 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 			}
 		}
 	} catch (Error& e) {
+		if (DEBUG_CF_CLIENT_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorError", debugUID).errorUnsuppressed(e);
+		}
 		if (e.code() == error_code_actor_cancelled) {
 			throw;
 		}
@@ -7775,6 +7781,7 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	TEST(interfs.size() > 10); // Large change feed merge cursor
 	TEST(interfs.size() > 100); // Very large change feed merge cursor
 
+	state std::vector<UID> debugUIDs;
 	results->streams.clear();
 	for (auto& it : interfs) {
 		ChangeFeedStreamRequest req;
@@ -7790,6 +7797,17 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		}
 		// TODO REMOVE
 		req.debugUID = deterministicRandom()->randomUniqueID();
+		debugUIDs.push_back(req.debugUID);
+
+		if (DEBUG_CF_CLIENT_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursor", req.debugUID)
+			    .detail("FeedID", rangeID)
+			    .detail("MergeRange", KeyRangeRef(interfs.front().second.begin, interfs.back().second.end))
+			    .detail("PartialRange", it.second)
+			    .detail("Begin", *begin)
+			    .detail("End", end)
+			    .detail("CanReadPopped", true);
+		}
 
 		results->streams.push_back(it.first.changeFeedStream.getReplyStream(req));
 	}
@@ -7811,8 +7829,14 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 
 	for (int i = 0; i < interfs.size(); i++) {
 		onErrors[i] = results->streams[i].onError();
-		fetchers[i] = partialChangeFeedStream(
-		    interfs[i].first, streams[i].results, results->streams[i], *begin, end, results, results->storageData[i]);
+		fetchers[i] = partialChangeFeedStream(interfs[i].first,
+		                                      streams[i].results,
+		                                      results->streams[i],
+		                                      *begin,
+		                                      end,
+		                                      results,
+		                                      results->storageData[i],
+		                                      debugUIDs[i]);
 	}
 
 	wait(onCFErrors(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end));
@@ -7940,6 +7964,15 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 	// TODO REMOVE
 	req.debugUID = deterministicRandom()->randomUniqueID();
 
+	if (DEBUG_CF_CLIENT_TRACE) {
+		TraceEvent(SevDebug, "TraceChangeFeedClientSingleCursor", req.debugUID)
+		    .detail("FeedID", rangeID)
+		    .detail("Range", range)
+		    .detail("Begin", *begin)
+		    .detail("End", end)
+		    .detail("CanReadPopped", true);
+	}
+
 	results->streams.clear();
 
 	for (auto& it : results->storageData) {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 86fba6190b..a4f5b9033c 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2333,7 +2333,7 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 
 	try {
 		if (DEBUG_CF_TRACE) {
-			TraceEvent(SevDebug, "TraceChangeFeedMutationsBegin", data->thisServerID)
+			TraceEvent(SevDebug, "TraceChangeFeedStreamStart", data->thisServerID)
 			    .detail("FeedID", req.rangeID)
 			    .detail("StreamUID", streamUID)
 			    .detail("Range", req.range)
@@ -2352,11 +2352,32 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 		ASSERT(emptyInitialReply.minStreamVersion == invalidVersion);
 		req.reply.send(emptyInitialReply);
 
+		if (DEBUG_CF_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedStreamSentInitialEmpty", data->thisServerID)
+			    .detail("FeedID", req.rangeID)
+			    .detail("StreamUID", streamUID)
+			    .detail("Range", req.range)
+			    .detail("Begin", req.begin)
+			    .detail("End", req.end)
+			    .detail("CanReadPopped", req.canReadPopped)
+			    .detail("Version", req.begin - 1);
+		}
+
 		loop {
 			Future<Void> onReady = req.reply.onReady();
 			if (atLatest && !onReady.isReady() && !removeUID) {
 				data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()][streamUID] =
 				    blockedVersion.present() ? blockedVersion.get() : data->prevVersion;
+				if (DEBUG_CF_TRACE) {
+					TraceEvent(SevDebug, "TraceChangeFeedStreamBlockedOnReady", data->thisServerID)
+					    .detail("FeedID", req.rangeID)
+					    .detail("StreamUID", streamUID)
+					    .detail("Range", req.range)
+					    .detail("Begin", req.begin)
+					    .detail("End", req.end)
+					    .detail("CanReadPopped", req.canReadPopped)
+					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion);
+				}
 				removeUID = true;
 			}
 			wait(onReady);
@@ -2367,6 +2388,16 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 				data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()][streamUID] =
 				    blockedVersion.present() ? blockedVersion.get() : data->prevVersion;
 				removeUID = true;
+				if (DEBUG_CF_TRACE) {
+					TraceEvent(SevDebug, "TraceChangeFeedStreamBlockedMutations", data->thisServerID)
+					    .detail("FeedID", req.rangeID)
+					    .detail("StreamUID", streamUID)
+					    .detail("Range", req.range)
+					    .detail("Begin", req.begin)
+					    .detail("End", req.end)
+					    .detail("CanReadPopped", req.canReadPopped)
+					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion);
+				}
 			}
 			std::pair<ChangeFeedStreamReply, bool> _feedReply = wait(feedReplyFuture);
 			ChangeFeedStreamReply feedReply = _feedReply.first;

From a2bbb188c3636dd42ee8a5d1c4c3d1a8f2beba84 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Fri, 11 Mar 2022 16:50:23 -0600
Subject: [PATCH 200/210] Fixing delayed health updater to mark address failed
 when too many connections triggers before new connection starts, instead of
 after

---
 fdbrpc/FlowTransport.actor.cpp | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 3e3b13709a..f4fbf461f0 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -545,28 +545,20 @@ ACTOR Future<Void> connectionWriter(Reference<Peer> self, Reference<IConnection>
 	}
 }
 
-ACTOR Future<Void> delayedHealthUpdate(NetworkAddress address) {
+ACTOR Future<Void> delayedHealthUpdate(NetworkAddress address, bool* tooManyConnectionsClosed) {
 	state double start = now();
-	state bool delayed = false;
 	loop {
 		if (FLOW_KNOBS->HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS &&
 		    FlowTransport::transport().healthMonitor()->tooManyConnectionsClosed(address) && address.isPublic()) {
-			if (!delayed) {
-				TraceEvent("TooManyConnectionsClosedMarkFailed")
-				    .detail("Dest", address)
-				    .detail("StartTime", start)
-				    .detail("ClosedCount", FlowTransport::transport().healthMonitor()->closedConnectionsCount(address));
-				IFailureMonitor::failureMonitor().setStatus(address, FailureStatus(true));
-			}
-			delayed = true;
 			wait(delayJittered(FLOW_KNOBS->MAX_RECONNECTION_TIME * 2.0));
 		} else {
-			if (delayed) {
+			if (*tooManyConnectionsClosed) {
 				TraceEvent("TooManyConnectionsClosedMarkAvailable")
 				    .detail("Dest", address)
 				    .detail("StartTime", start)
 				    .detail("TimeElapsed", now() - start)
 				    .detail("ClosedCount", FlowTransport::transport().healthMonitor()->closedConnectionsCount(address));
+				*tooManyConnectionsClosed = false;
 			}
 			IFailureMonitor::failureMonitor().setStatus(address, FailureStatus(false));
 			break;
@@ -586,6 +578,7 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 	state Future<Void> delayedHealthUpdateF;
 	state Optional<double> firstConnFailedTime = Optional<double>();
 	state int retryConnect = false;
+	state bool tooManyConnectionsClosed = false;
 
 	loop {
 		try {
@@ -635,7 +628,8 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 								IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(false));
 							}
 							if (self->unsent.empty()) {
-								delayedHealthUpdateF = delayedHealthUpdate(self->destination);
+								delayedHealthUpdateF =
+								    delayedHealthUpdate(self->destination, &tooManyConnectionsClosed);
 								choose {
 									when(wait(delayedHealthUpdateF)) {
 										conn->close();
@@ -675,7 +669,7 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 			try {
 				self->transport->countConnEstablished++;
 				if (!delayedHealthUpdateF.isValid())
-					delayedHealthUpdateF = delayedHealthUpdate(self->destination);
+					delayedHealthUpdateF = delayedHealthUpdate(self->destination, &tooManyConnectionsClosed);
 				wait(connectionWriter(self, conn) || reader || connectionMonitor(self) ||
 				     self->resetConnection.onTrigger());
 				TraceEvent("ConnectionReset", conn ? conn->getDebugID() : UID())
@@ -761,6 +755,17 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 			if (conn) {
 				if (self->destination.isPublic() && e.code() == error_code_connection_failed) {
 					FlowTransport::transport().healthMonitor()->reportPeerClosed(self->destination);
+					if (FLOW_KNOBS->HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS &&
+					    FlowTransport::transport().healthMonitor()->tooManyConnectionsClosed(self->destination) &&
+					    self->destination.isPublic()) {
+						TraceEvent("TooManyConnectionsClosedMarkFailed")
+						    .detail("Dest", self->destination)
+						    .detail(
+						        "ClosedCount",
+						        FlowTransport::transport().healthMonitor()->closedConnectionsCount(self->destination));
+						tooManyConnectionsClosed = true;
+						IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(true));
+					}
 				}
 
 				conn->close();

From 26cbe6863d7f37d71bb613e000843abc7dfda7c5 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 11:48:15 -0500
Subject: [PATCH 201/210] Adjusting simulation knobs to reduce change feed
 memory overhead for large merge cases

---
 fdbclient/ServerKnobs.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index bc6f181c79..620b24cd2f 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -290,7 +290,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SQLITE_CHUNK_SIZE_PAGES_SIM,                          1024 );  // 4MB
 	init( SQLITE_READER_THREADS,                                  64 );  // number of read threads
 	init( SQLITE_WRITE_WINDOW_SECONDS,                            -1 );
-	init( SQLITE_CURSOR_MAX_LIFETIME_BYTES,                      1e6 ); if( randomize && BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0;
+	init( SQLITE_CURSOR_MAX_LIFETIME_BYTES,                      1e6 ); if (buggifySmallShards || simulationMediumShards) SQLITE_CURSOR_MAX_LIFETIME_BYTES = MIN_SHARD_BYTES; if( randomize && BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0;
 	init( SQLITE_WRITE_WINDOW_LIMIT,                              -1 );
 	if( randomize && BUGGIFY ) {
 		// Choose an window between .01 and 1.01 seconds.
@@ -822,7 +822,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	// Blob granlues
 	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually
-	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards || (randomize && BUGGIFY) ) { deterministicRandom()->random01() < 0.1 ? BG_SNAPSHOT_FILE_TARGET_BYTES /= 100 : BG_SNAPSHOT_FILE_TARGET_BYTES /= 10; }
+	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (simulationMediumShards || (randomize && BUGGIFY) ) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000; 
 	init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 );
 	init( BG_DELTA_FILE_TARGET_BYTES,   BG_DELTA_BYTES_BEFORE_COMPACT/10 );
 	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15);

From 98eeaac1b7c2595f7f30510436c5b6873517ad41 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 11:57:43 -0500
Subject: [PATCH 202/210] More debugging and fixing a stuck merge cursor

---
 fdbclient/DatabaseContext.h   |   2 +
 fdbclient/NativeAPI.actor.cpp | 133 ++++++++++++++++++++++++++++++----
 2 files changed, 121 insertions(+), 14 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 485617c102..65b47f95f9 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -170,6 +170,8 @@ public:
 struct MutationAndVersionStream {
 	Standalone<MutationsAndVersionRef> next;
 	PromiseStream<Standalone<MutationsAndVersionRef>> results;
+	// TODO REMOVE, for debugging
+	int idx = -1;
 	bool operator<(MutationAndVersionStream const& rhs) const { return next.version > rhs.next.version; }
 };
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 1e3c70a733..e82c767dcd 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7479,6 +7479,9 @@ Version ChangeFeedData::getVersion() {
 	return lastReturnedVersion.get();
 }
 
+// TODO REMOVE
+#define DEBUG_CF_WAIT_VERSION invalidVersion
+
 // This function is essentially bubbling the information about what has been processed from the server through the
 // change feed client. First it makes sure the server has returned all mutations up through the target version, the
 // native api has consumed and processed, them, and then the fdb client has consumed all of the mutations.
@@ -7486,11 +7489,28 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 	// wait on SS to have sent up through version
 	int desired = 0;
 	int waiting = 0;
+	if (version == DEBUG_CF_WAIT_VERSION) {
+		fmt::print("WFV {}) waitLatest\n", version);
+	}
 	std::vector<Future<Void>> allAtLeast;
 	for (auto& it : self->storageData) {
 		if (it->version.get() < version) {
 			waiting++;
+			if (version == DEBUG_CF_WAIT_VERSION) {
+				fmt::print("WFV {0})   SS {1} {2} < {3}\n",
+				           version,
+				           it->id.toString().substr(0, 4),
+				           it->version.get(),
+				           version);
+			}
 			if (version > it->desired.get()) {
+				if (version == DEBUG_CF_WAIT_VERSION) {
+					fmt::print("WFV {0})   SS {1} desired {2} -> {3}\n",
+					           version,
+					           it->id.toString().substr(0, 4),
+					           it->desired.get(),
+					           version);
+				}
 				it->desired.set(version);
 				desired++;
 			}
@@ -7498,6 +7518,10 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 		}
 	}
 
+	if (version == DEBUG_CF_WAIT_VERSION) {
+		fmt::print("WFV {0}) waiting for {1}/{2} ({3} desired)\n", version, waiting, self->storageData.size(), desired);
+	}
+
 	wait(waitForAll(allAtLeast));
 
 	// then, wait on ss streams to have processed up through version
@@ -7547,7 +7571,9 @@ ACTOR Future<Void> changeFeedWhenAtLatest(Reference<ChangeFeedData> self, Versio
 		return Void();
 	}
 	state Future<Void> lastReturned = self->lastReturnedVersion.whenAtLeast(version);
-
+	if (version == DEBUG_CF_WAIT_VERSION) {
+		fmt::print("WFV {0}) notAtLatest={1}\n", version, self->notAtLatest.get());
+	}
 	loop {
 		// only allowed to use empty versions if you're caught up
 		Future<Void> waitEmptyVersion = (self->notAtLatest.get() == 0) ? changeFeedWaitLatest(self, version) : Never();
@@ -7570,7 +7596,7 @@ Future<Void> ChangeFeedData::whenAtLeast(Version version) {
 	return changeFeedWhenAtLatest(Reference<ChangeFeedData>::addRef(this), version);
 }
 
-#define DEBUG_CF_CLIENT_TRACE false
+#define DEBUG_CF_CLIENT_TRACE true
 
 ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
                                            PromiseStream<Standalone<MutationsAndVersionRef>> results,
@@ -7585,6 +7611,12 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 	state Promise<Void> refresh = feedData->refresh;
 	state bool atLatestVersion = false;
 	state Version nextVersion = begin;
+	// We don't need to force every other partial stream to do an empty if we get an empty, but if we get actual
+	// mutations back after sending an empty, we may need the other partial streams to get an empty, to advance the
+	// merge cursor, so we can send the mutations we just got.
+	// if lastEmpty != invalidVersion, we need to update the desired versions of the other streams BEFORE waiting
+	// onReady once getting a reply
+	state Version lastEmpty = invalidVersion;
 	try {
 		loop {
 			if (nextVersion >= end) {
@@ -7599,6 +7631,18 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 						continue;
 					}
 
+					if (DEBUG_CF_CLIENT_TRACE) {
+						TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorReply", debugUID)
+						    .detail("SSID", storageData->id)
+						    .detail("AtLatest", atLatestVersion)
+						    .detail("FirstVersion", rep.mutations.front().version)
+						    .detail("LastVersion", rep.mutations.back().version)
+						    .detail("Count", rep.mutations.size())
+						    .detail("MinStreamVersion", rep.minStreamVersion)
+						    .detail("PopVersion", rep.popVersion)
+						    .detail("RepAtLatest", rep.atLatestVersion);
+					}
+
 					if (rep.mutations.back().version > feedData->maxSeenVersion) {
 						feedData->maxSeenVersion = rep.mutations.back().version;
 					}
@@ -7606,12 +7650,27 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 						feedData->popVersion = rep.popVersion;
 					}
 
+					if (lastEmpty != invalidVersion && !results.isEmpty()) {
+						for (auto& it : feedData->storageData) {
+							if (refresh.canBeSet() && lastEmpty > it->desired.get()) {
+								it->desired.set(lastEmpty);
+							}
+						}
+						lastEmpty = invalidVersion;
+					}
+
 					state int resultLoc = 0;
 					while (resultLoc < rep.mutations.size()) {
 						wait(results.onEmpty());
 						if (rep.mutations[resultLoc].version >= nextVersion) {
 							results.send(rep.mutations[resultLoc]);
 
+							if (DEBUG_CF_CLIENT_TRACE) {
+								TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSend", debugUID)
+								    .detail("Version", rep.mutations[resultLoc].version)
+								    .detail("Size", rep.mutations[resultLoc].mutations.size());
+							}
+
 							// check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we
 							// just skip to the next wait and get actor_cancelled
 							// FIXME: this is somewhat expensive to do every mutation.
@@ -7638,6 +7697,10 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 					if (refresh.canBeSet() && rep.minStreamVersion > storageData->version.get()) {
 						storageData->version.set(rep.minStreamVersion);
 					}
+					if (DEBUG_CF_CLIENT_TRACE) {
+						TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorReplyDone", debugUID)
+						    .detail("AtLatestNow", atLatestVersion);
+					}
 				}
 				when(wait(atLatestVersion && replyStream.isEmpty() && results.isEmpty()
 				              ? storageData->version.whenAtLeast(nextVersion)
@@ -7646,6 +7709,11 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 					empty.version = storageData->version.get();
 					results.send(empty);
 					nextVersion = storageData->version.get() + 1;
+					if (DEBUG_CF_CLIENT_TRACE) {
+						TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSendEmpty", debugUID)
+						    .detail("Version", empty.version);
+					}
+					lastEmpty = empty.version;
 				}
 				when(wait(atLatestVersion && replyStream.isEmpty() && !results.isEmpty() ? results.onEmpty()
 				                                                                         : Future<Void>(Never()))) {}
@@ -7667,18 +7735,30 @@ ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> resul
                                                  std::vector<std::pair<StorageServerInterface, KeyRange>> interfs,
                                                  std::vector<MutationAndVersionStream> streams,
                                                  Version* begin,
-                                                 Version end) {
+                                                 Version end,
+                                                 UID mergeCursorUID) {
 	state Promise<Void> refresh = results->refresh;
 	// with empty version handling in the partial cursor, all streams will always have a next element with version >=
 	// the minimum version of any stream's next element
 	state std::priority_queue<MutationAndVersionStream, std::vector<MutationAndVersionStream>> mutations;
 
+	if (DEBUG_CF_CLIENT_TRACE) {
+		TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorStart", mergeCursorUID)
+		    .detail("StreamCount", interfs.size())
+		    .detail("Begin", *begin)
+		    .detail("End", end);
+	}
+
 	// previous version of change feed may have put a mutation in the promise stream and then immediately died. Wait for
 	// that mutation first, so the promise stream always starts empty
 	wait(results->mutations.onEmpty());
 	wait(delay(0));
 	ASSERT(results->mutations.isEmpty());
 
+	if (DEBUG_CF_CLIENT_TRACE) {
+		TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorGotEmpty", mergeCursorUID);
+	}
+
 	// update lastReturned once the previous mutation has been consumed
 	if (*begin - 1 > results->lastReturnedVersion.get()) {
 		results->lastReturnedVersion.set(*begin - 1);
@@ -7698,13 +7778,26 @@ ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> resul
 		interfNum = 0;
 		while (interfNum < streamsUsed.size()) {
 			try {
+				if (DEBUG_CF_CLIENT_TRACE) {
+					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorWait", mergeCursorUID)
+					    .detail("Idx", streamsUsed[interfNum].idx);
+				}
 				Standalone<MutationsAndVersionRef> res = waitNext(streamsUsed[interfNum].results.getFuture());
 				streamsUsed[interfNum].next = res;
 				mutations.push(streamsUsed[interfNum]);
+				if (DEBUG_CF_CLIENT_TRACE) {
+					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorGot", mergeCursorUID)
+					    .detail("Idx", streamsUsed[interfNum].idx)
+					    .detail("Version", res.version);
+				}
 			} catch (Error& e) {
 				if (e.code() != error_code_end_of_stream) {
 					throw e;
 				}
+				if (DEBUG_CF_CLIENT_TRACE) {
+					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorEOS", mergeCursorUID)
+					    .detail("Idx", streamsUsed[interfNum].idx);
+				}
 			}
 			interfNum++;
 		}
@@ -7742,6 +7835,12 @@ ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> resul
 
 		*begin = nextVersion + 1;
 
+		if (DEBUG_CF_CLIENT_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSending", mergeCursorUID)
+			    .detail("Count", streamsUsed.size())
+			    .detail("Version", nextVersion);
+		}
+
 		// send mutations at nextVersion to the client
 		if (nextOut.back().mutations.empty()) {
 			ASSERT(results->mutations.isEmpty());
@@ -7781,6 +7880,8 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	TEST(interfs.size() > 10); // Large change feed merge cursor
 	TEST(interfs.size() > 100); // Very large change feed merge cursor
 
+	// TODO REMOVE eventually, useful for debugging for now
+	state UID mergeCursorUID = UID();
 	state std::vector<UID> debugUIDs;
 	results->streams.clear();
 	for (auto& it : interfs) {
@@ -7798,16 +7899,8 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		// TODO REMOVE
 		req.debugUID = deterministicRandom()->randomUniqueID();
 		debugUIDs.push_back(req.debugUID);
-
-		if (DEBUG_CF_CLIENT_TRACE) {
-			TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursor", req.debugUID)
-			    .detail("FeedID", rangeID)
-			    .detail("MergeRange", KeyRangeRef(interfs.front().second.begin, interfs.back().second.end))
-			    .detail("PartialRange", it.second)
-			    .detail("Begin", *begin)
-			    .detail("End", end)
-			    .detail("CanReadPopped", true);
-		}
+		mergeCursorUID =
+		    UID(mergeCursorUID.first() ^ req.debugUID.first(), mergeCursorUID.second() ^ req.debugUID.second());
 
 		results->streams.push_back(it.first.changeFeedStream.getReplyStream(req));
 	}
@@ -7828,6 +7921,18 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	refresh.send(Void());
 
 	for (int i = 0; i < interfs.size(); i++) {
+		if (DEBUG_CF_CLIENT_TRACE) {
+			TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorInit", debugUIDs[i])
+			    .detail("CursorDebugUID", mergeCursorUID)
+			    .detail("Idx", i)
+			    .detail("FeedID", rangeID)
+			    .detail("MergeRange", KeyRangeRef(interfs.front().second.begin, interfs.back().second.end))
+			    .detail("PartialRange", interfs[i].second)
+			    .detail("Begin", *begin)
+			    .detail("End", end)
+			    .detail("CanReadPopped", canReadPopped);
+		}
+		streams[i].idx = i;
 		onErrors[i] = results->streams[i].onError();
 		fetchers[i] = partialChangeFeedStream(interfs[i].first,
 		                                      streams[i].results,
@@ -7839,7 +7944,7 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		                                      debugUIDs[i]);
 	}
 
-	wait(onCFErrors(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end));
+	wait(onCFErrors(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end, mergeCursorUID));
 
 	return Void();
 }

From 8d88b7ca41c50ead42f9333b30c5050f971b1d4b Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 14:39:00 -0500
Subject: [PATCH 203/210] making range assigner boundary changes fully
 synchronous to ensure no races with things reading assignments

---
 fdbclient/NativeAPI.actor.cpp   |  2 +-
 fdbserver/BlobManager.actor.cpp | 95 ++++++++++++++++++++-------------
 2 files changed, 60 insertions(+), 37 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index e82c767dcd..07d983cd64 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -7596,7 +7596,7 @@ Future<Void> ChangeFeedData::whenAtLeast(Version version) {
 	return changeFeedWhenAtLatest(Reference<ChangeFeedData>::addRef(this), version);
 }
 
-#define DEBUG_CF_CLIENT_TRACE true
+#define DEBUG_CF_CLIENT_TRACE false
 
 ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
                                            PromiseStream<Standalone<MutationsAndVersionRef>> results,
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 9a2e1cdb69..712e069dee 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -342,6 +342,8 @@ ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
 		}
 		bmData->restartRecruiting.trigger();
 		wait(bmData->recruitingStream.onChange() || bmData->foundBlobWorkers.getFuture());
+		// FIXME: may want to have some buffer here so zero-worker recruiting case doesn't assign every single pending
+		// range to the first worker recruited
 	}
 
 	int minGranulesAssigned = INT_MAX;
@@ -374,8 +376,23 @@ ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
 
 ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
                                      RangeAssignment assignment,
-                                     UID workerID,
+                                     Optional<UID> workerID,
                                      int64_t seqNo) {
+	// WorkerId is set, except in case of assigning to any worker. Then we pick the worker to assign to in here
+
+	// inject delay into range assignments
+	if (BUGGIFY_WITH_PROB(0.05)) {
+		wait(delay(deterministicRandom()->random01()));
+	}
+
+	if (!workerID.present()) {
+		ASSERT(assignment.isAssign);
+		UID _workerId = wait(pickWorkerForAssign(bmData));
+		if (BM_DEBUG) {
+			fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
+		}
+		workerID = _workerId;
+	}
 
 	if (BM_DEBUG) {
 		fmt::print("BM {0} {1} range [{2} - {3}) @ ({4}, {5}) to {6}\n",
@@ -385,7 +402,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 		           assignment.keyRange.end.printable(),
 		           bmData->epoch,
 		           seqNo,
-		           workerID.toString());
+		           workerID.get().toString());
 	}
 
 	try {
@@ -401,10 +418,10 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			req.type = assignment.assign.get().type;
 
 			// if that worker isn't alive anymore, add the range back into the stream
-			if (bmData->workersById.count(workerID) == 0) {
+			if (bmData->workersById.count(workerID.get()) == 0) {
 				throw no_more_servers();
 			}
-			wait(bmData->workersById[workerID].assignBlobRangeRequest.getReply(req));
+			wait(bmData->workersById[workerID.get()].assignBlobRangeRequest.getReply(req));
 		} else {
 			ASSERT(!assignment.assign.present());
 			ASSERT(assignment.revoke.present());
@@ -417,8 +434,8 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			req.dispose = assignment.revoke.get().dispose;
 
 			// if that worker isn't alive anymore, this is a noop
-			if (bmData->workersById.count(workerID)) {
-				wait(bmData->workersById[workerID].revokeBlobRangeRequest.getReply(req));
+			if (bmData->workersById.count(workerID.get())) {
+				wait(bmData->workersById[workerID.get()].revokeBlobRangeRequest.getReply(req));
 			} else {
 				return Void();
 			}
@@ -443,7 +460,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 				           bmData->epoch,
 				           assignment.keyRange.begin.printable(),
 				           assignment.keyRange.end.printable(),
-				           workerID.toString());
+				           workerID.get().toString());
 			}
 			if (bmData->doLockCheck.canBeSet()) {
 				bmData->doLockCheck.send(Void());
@@ -472,7 +489,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 				           e.name(),
 				           assignment.keyRange.begin.printable(),
 				           assignment.keyRange.end.printable(),
-				           workerID.toString());
+				           workerID.get().toString());
 			}
 
 			// re-send revoke to queue to handle range being un-assigned from that worker before the new one
@@ -518,10 +535,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 
 ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 	loop {
-		// inject delay into range assignments
-		if (BUGGIFY_WITH_PROB(0.05)) {
-			wait(delay(deterministicRandom()->random01()));
-		}
+
 		state RangeAssignment assignment = waitNext(bmData->rangesToAssign.getFuture());
 		state int64_t seqNo = bmData->seqNo;
 		bmData->seqNo++;
@@ -568,22 +582,22 @@ ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 					           bmData->id.toString());
 				}
 				workerId = assignment.worker.get();
+
+				bmData->workerAssignments.insert(assignment.keyRange, workerId);
+				bmData->assignsInProgress.insert(assignment.keyRange,
+				                                 doRangeAssignment(bmData, assignment, workerId, seqNo));
 			} else {
-				UID _workerId = wait(pickWorkerForAssign(bmData));
-				if (BM_DEBUG) {
-					fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
-				}
-				workerId = _workerId;
+				// Ensure the key boundaries are updated before we pick a worker
+				bmData->workerAssignments.insert(assignment.keyRange, UID());
+				bmData->assignsInProgress.insert(assignment.keyRange,
+				                                 doRangeAssignment(bmData, assignment, Optional<UID>(), seqNo));
 			}
-			bmData->workerAssignments.insert(assignment.keyRange, workerId);
 
 			// If we know about the worker and this is not a continue, then this is a new range for the worker
 			if (bmData->workerStats.count(workerId) && assignment.assign.get().type != AssignRequestType::Continue) {
 				bmData->workerStats[workerId].numGranulesAssigned += 1;
 			}
 
-			bmData->assignsInProgress.insert(assignment.keyRange,
-			                                 doRangeAssignment(bmData, assignment, workerId, seqNo));
 		} else {
 			if (assignment.worker.present()) {
 				// revoke this specific range from this specific worker. Either part of recovery or failing a worker
@@ -778,7 +792,7 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 						ra.assign = RangeAssignmentData(); // type=normal
 						bmData->rangesToAssign.send(ra);
 					}
-					wait(bmData->rangesToAssign.onEmpty());
+					ASSERT(bmData->rangesToAssign.isEmpty());
 				}
 
 				lastChangeKeyValue =
@@ -956,6 +970,23 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				                      existingState.size() + 2));
 				// +2 because this is boundaries and existingState was granules, and to ensure it doesn't set more
 				ASSERT(!existingBoundaries.more);
+				// TODO remove debugging printf
+				if (existingBoundaries.size() != existingState.size() + 1) {
+					printf("DBG: EB=%d, ES=%d for [%s - %s)\n",
+					       existingBoundaries.size(),
+					       existingState.size(),
+					       granuleRange.begin.printable().c_str(),
+					       granuleRange.end.printable().c_str());
+					printf("Boundaries:\n");
+					for (auto& it : existingBoundaries) {
+						printf("  %s\n", it.key.removePrefix(blobGranuleMappingKeys.begin).printable().c_str());
+					}
+					printf("State:\n");
+					for (auto& it : existingState) {
+						std::pair<UID, UID> k = decodeBlobGranuleSplitKey(it.key);
+						printf("  %s\n", k.second.toString().substr(0, 6).c_str());
+					}
+				}
 				ASSERT(existingBoundaries.size() == existingState.size() + 1);
 				newRanges.clear();
 				newRanges.arena().dependsOn(existingBoundaries.arena());
@@ -1100,18 +1131,9 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		bmData->rangesToAssign.send(raAssignSplit);
 	}
 
-	if (BM_DEBUG) {
-		fmt::print(
-		    "Splitting range [{0} - {1}) into {2} granules @ {3} sent assignments, waiting for them to be processed:\n",
-		    granuleRange.begin.printable(),
-		    granuleRange.end.printable(),
-		    newRanges.size() - 1,
-		    latestVersion);
-	}
-
 	// Ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
 	// before returning. This prevents a race with a subsequent split evaluation
-	wait(bmData->rangesToAssign.onEmpty());
+	ASSERT(bmData->rangesToAssign.isEmpty());
 
 	if (BM_DEBUG) {
 		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} got assignments processed\n",
@@ -1221,6 +1243,8 @@ ACTOR Future<Void> killBlobWorker(Reference<BlobManagerData> bmData, BlobWorkerI
 		bmData->rangesToAssign.send(raAssign);
 	}
 
+	ASSERT(bmData->rangesToAssign.isEmpty());
+
 	// Send halt to blob worker, with no expectation of hearing back
 	if (BM_DEBUG) {
 		fmt::print("Sending halt to BW {}\n", bwId.toString());
@@ -1229,10 +1253,7 @@ ACTOR Future<Void> killBlobWorker(Reference<BlobManagerData> bmData, BlobWorkerI
 
 	// wait for blob worker to be removed from DB and in-memory mapping to have reassigned all shards from this worker
 	// before removing it from deadWorkers, to avoid a race with checkBlobWorkerList
-	wait(deregister && bmData->rangesToAssign.onEmpty());
-	// delay(0) after onEmpty to yield back to the range assigner on the final pop to ensure it gets processed before
-	// deadWorkers.erase
-	wait(delay(0));
+	wait(deregister);
 
 	// restart recruiting to replace the dead blob worker
 	bmData->restartRecruiting.trigger();
@@ -2686,9 +2707,12 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 
 	self->epoch = epoch;
 
+	// start rangeAssigner first since other actors can send messages to it
+	self->addActor.send(rangeAssigner(self));
 	// although we start the recruiter, we wait until existing workers are ack'd
 	auto recruitBlobWorker = IAsyncListener<RequestStream<RecruitBlobWorkerRequest>>::create(
 	    dbInfo, [](auto const& info) { return info.clusterInterface.recruitBlobWorker; });
+
 	self->addActor.send(blobWorkerRecruiter(self, recruitBlobWorker));
 
 	// we need to recover the old blob manager's state (e.g. granule assignments) before
@@ -2697,7 +2721,6 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 
 	self->addActor.send(doLockChecks(self));
 	self->addActor.send(monitorClientRanges(self));
-	self->addActor.send(rangeAssigner(self));
 	self->addActor.send(monitorPruneKeys(self));
 
 	if (BUGGIFY) {

From e7aa534cae1a5d69f607020ab74ef8d837155c57 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 15:04:43 -0500
Subject: [PATCH 204/210] Forgot to update worker mapping after picking worker

---
 fdbserver/BlobManager.actor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 712e069dee..fb1293e3dc 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -392,6 +392,9 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
 		}
 		workerID = _workerId;
+		// We don't have to check for races with an overlapping assignment because it would insert over us in the actor
+		// map, cancelling this actor before it got here
+		bmData->workerAssignments.insert(assignment.keyRange, workerID.get());
 	}
 
 	if (BM_DEBUG) {

From 3e278e8580b0acf131ee30f1bc97c57af30c1cd6 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 15:53:09 -0500
Subject: [PATCH 205/210] asserts for assign is empty is false if BM dies and
 rangeAssigner actor is dead

---
 fdbserver/BlobManager.actor.cpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index fb1293e3dc..916c9b25a3 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -795,7 +795,6 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 						ra.assign = RangeAssignmentData(); // type=normal
 						bmData->rangesToAssign.send(ra);
 					}
-					ASSERT(bmData->rangesToAssign.isEmpty());
 				}
 
 				lastChangeKeyValue =
@@ -1134,10 +1133,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		bmData->rangesToAssign.send(raAssignSplit);
 	}
 
-	// Ensure the new assignments actually got processed and the split boundaries are reflected in the granule mapping
-	// before returning. This prevents a race with a subsequent split evaluation
-	ASSERT(bmData->rangesToAssign.isEmpty());
-
 	if (BM_DEBUG) {
 		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} got assignments processed\n",
 		           granuleRange.begin.printable(),
@@ -1246,8 +1241,6 @@ ACTOR Future<Void> killBlobWorker(Reference<BlobManagerData> bmData, BlobWorkerI
 		bmData->rangesToAssign.send(raAssign);
 	}
 
-	ASSERT(bmData->rangesToAssign.isEmpty());
-
 	// Send halt to blob worker, with no expectation of hearing back
 	if (BM_DEBUG) {
 		fmt::print("Sending halt to BW {}\n", bwId.toString());

From 1c14d4674dd6223dfd622c894b2c4852592f0fae Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 16:11:17 -0500
Subject: [PATCH 206/210] Fixing more BM races with granule splitting

---
 fdbserver/BlobManager.actor.cpp | 36 +++++++--------------------------
 1 file changed, 7 insertions(+), 29 deletions(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 916c9b25a3..7943f948d8 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -865,7 +865,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
                                    bool writeHot) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 	state Standalone<VectorRef<KeyRef>> newRanges;
-	state int64_t newLockSeqno = -1;
 
 	// first get ranges to split
 	Standalone<VectorRef<KeyRef>> _newRanges = wait(splitRange(bmData, granuleRange, writeHot));
@@ -1015,7 +1014,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			ASSERT(lockValue.present());
 			std::tuple<int64_t, int64_t, UID> prevGranuleLock = decodeBlobGranuleLockValue(lockValue.get());
 			int64_t ownerEpoch = std::get<0>(prevGranuleLock);
-			int64_t ownerSeqno = std::get<1>(prevGranuleLock);
 
 			if (ownerEpoch > bmData->epoch) {
 				if (BM_DEBUG) {
@@ -1033,33 +1031,13 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				return Void();
 			}
 
-			if (newLockSeqno == -1) {
-				newLockSeqno = bmData->seqNo;
-				bmData->seqNo++;
-				if (!(bmData->epoch > ownerEpoch || (bmData->epoch == ownerEpoch && newLockSeqno > ownerSeqno))) {
-					fmt::print("BM seqno for granule [{0} - {1}) out of order for lock! manager: ({2}, {3}), owner: "
-					           "({4}, {5}})\n",
-					           granuleRange.begin.printable(),
-					           granuleRange.end.printable(),
-					           bmData->epoch,
-					           newLockSeqno,
-					           ownerEpoch,
-					           ownerSeqno);
-				}
-				ASSERT(bmData->epoch > ownerEpoch || (bmData->epoch == ownerEpoch && newLockSeqno > ownerSeqno));
-			} else if (bmData->epoch == ownerEpoch && newLockSeqno < ownerSeqno) {
-				// we retried, and between retries we reassigned this range elsewhere. Cancel this split
-				TEST(true); // BM maybe split cancelled by subsequent move
-				if (BM_DEBUG) {
-					fmt::print("Splitting range [{0} - {1}) cancelled by move elsewhere!\n",
-					           granuleRange.begin.printable(),
-					           granuleRange.end.printable());
-				}
-				return Void();
-			}
-
-			// acquire granule lock so nobody else can make changes to this granule.
-			tr->set(lockKey, blobGranuleLockValueFor(bmData->epoch, newLockSeqno, std::get<2>(prevGranuleLock)));
+			// Set lock to max value for this manager, so other reassignments can't race with this transaction
+			// and existing owner can't modify it further.
+			// FIXME: Implementing merging may require us to make lock go backwards if we later merge other granules
+			// back to this same range, but I think that's fine
+			tr->set(lockKey,
+			        blobGranuleLockValueFor(
+			            bmData->epoch, std::numeric_limits<int64_t>::max(), std::get<2>(prevGranuleLock)));
 
 			// set up splits in granule mapping, but point each part to the old owner (until they get reassigned)
 			state int i;

From 8c35ceb68462fa58bd4e7e74a7d1649a1154dc08 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 17:38:31 -0500
Subject: [PATCH 207/210] switching bg file keys to sort on version then type,
 and using that to fix race where 2 blob workers try to split the same granule
 at different versions

---
 fdbclient/BlobWorkerInterface.h             |  9 +--
 fdbclient/SystemData.cpp                    | 12 ++--
 fdbclient/SystemData.h                      |  4 +-
 fdbserver/BlobGranuleServerCommon.actor.cpp |  2 +-
 fdbserver/BlobManager.actor.cpp             | 68 +++++++++++++--------
 fdbserver/BlobWorker.actor.cpp              | 13 ++--
 6 files changed, 64 insertions(+), 44 deletions(-)

diff --git a/fdbclient/BlobWorkerInterface.h b/fdbclient/BlobWorkerInterface.h
index 29318d6a1a..97d97e547a 100644
--- a/fdbclient/BlobWorkerInterface.h
+++ b/fdbclient/BlobWorkerInterface.h
@@ -152,7 +152,6 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 	int64_t seqno;
 	UID granuleID;
 	Version startVersion;
-	Version latestVersion;
 
 	GranuleStatusReply() {}
 	explicit GranuleStatusReply(KeyRange range,
@@ -161,10 +160,9 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 	                            int64_t epoch,
 	                            int64_t seqno,
 	                            UID granuleID,
-	                            Version startVersion,
-	                            Version latestVersion)
+	                            Version startVersion)
 	  : granuleRange(range), doSplit(doSplit), writeHotSplit(writeHotSplit), epoch(epoch), seqno(seqno),
-	    granuleID(granuleID), startVersion(startVersion), latestVersion(latestVersion) {}
+	    granuleID(granuleID), startVersion(startVersion) {}
 
 	int expectedSize() const { return sizeof(GranuleStatusReply) + granuleRange.expectedSize(); }
 
@@ -179,8 +177,7 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 		           epoch,
 		           seqno,
 		           granuleID,
-		           startVersion,
-		           latestVersion);
+		           startVersion);
 	}
 };
 
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index 2ed6872d57..21aea18d11 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -1133,26 +1133,26 @@ const KeyRef blobGranulePruneChangeKey = LiteralStringRef("\xff\x02/bgpChange");
 const uint8_t BG_FILE_TYPE_DELTA = 'D';
 const uint8_t BG_FILE_TYPE_SNAPSHOT = 'S';
 
-const Key blobGranuleFileKeyFor(UID granuleID, uint8_t fileType, Version fileVersion) {
+const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t fileType) {
 	ASSERT(fileType == 'D' || fileType == 'S');
 	BinaryWriter wr(AssumeVersion(ProtocolVersion::withBlobGranule()));
 	wr.serializeBytes(blobGranuleFileKeys.begin);
 	wr << granuleID;
-	wr << fileType;
 	wr << bigEndian64(fileVersion);
+	wr << fileType;
 	return wr.toValue();
 }
 
-std::tuple<UID, uint8_t, Version> decodeBlobGranuleFileKey(KeyRef const& key) {
+std::tuple<UID, Version, uint8_t> decodeBlobGranuleFileKey(KeyRef const& key) {
 	UID granuleID;
-	uint8_t fileType;
 	Version fileVersion;
+	uint8_t fileType;
 	BinaryReader reader(key.removePrefix(blobGranuleFileKeys.begin), AssumeVersion(ProtocolVersion::withBlobGranule()));
 	reader >> granuleID;
-	reader >> fileType;
 	reader >> fileVersion;
+	reader >> fileType;
 	ASSERT(fileType == 'D' || fileType == 'S');
-	return std::tuple(granuleID, fileType, bigEndian64(fileVersion));
+	return std::tuple(granuleID, bigEndian64(fileVersion), fileType);
 }
 
 const KeyRange blobGranuleFileKeyRangeFor(UID granuleID) {
diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h
index ca21eca61f..0facf79a0a 100644
--- a/fdbclient/SystemData.h
+++ b/fdbclient/SystemData.h
@@ -561,8 +561,8 @@ extern const KeyRangeRef blobGranulePruneKeys;
 extern const KeyRangeRef blobGranuleVersionKeys;
 extern const KeyRef blobGranulePruneChangeKey;
 
-const Key blobGranuleFileKeyFor(UID granuleID, uint8_t fileType, Version fileVersion);
-std::tuple<UID, uint8_t, Version> decodeBlobGranuleFileKey(KeyRef const& key);
+const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t fileType);
+std::tuple<UID, Version, uint8_t> decodeBlobGranuleFileKey(KeyRef const& key);
 const KeyRange blobGranuleFileKeyRangeFor(UID granuleID);
 
 const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length);
diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp
index 94192bb8d7..c47ed13199 100644
--- a/fdbserver/BlobGranuleServerCommon.actor.cpp
+++ b/fdbserver/BlobGranuleServerCommon.actor.cpp
@@ -59,7 +59,7 @@ ACTOR Future<Void> readGranuleFiles(Transaction* tr, Key* startKey, Key endKey,
 			int64_t offset;
 			int64_t length;
 
-			std::tie(gid, fileType, version) = decodeBlobGranuleFileKey(it.key);
+			std::tie(gid, version, fileType) = decodeBlobGranuleFileKey(it.key);
 			ASSERT(gid == granuleID);
 
 			std::tie(filename, offset, length) = decodeBlobGranuleFileValue(it.value);
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 7943f948d8..5568ede056 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -47,7 +47,7 @@
  * The Blob Manager is responsible for managing range granules, and recruiting and monitoring Blob Workers.
  */
 
-#define BM_DEBUG false
+#define BM_DEBUG true
 
 void handleClientBlobRange(KeyRangeMap<bool>* knownBlobRanges,
                            Arena& ar,
@@ -204,12 +204,11 @@ struct BlobWorkerStats {
 struct SplitEvaluation {
 	int64_t epoch;
 	int64_t seqno;
-	Version version;
 	Future<Void> inProgress;
 
-	SplitEvaluation() : epoch(0), seqno(0), version(invalidVersion) {}
-	SplitEvaluation(int64_t epoch, int64_t seqno, int64_t version, Future<Void> inProgress)
-	  : epoch(epoch), seqno(seqno), version(version), inProgress(inProgress) {}
+	SplitEvaluation() : epoch(0), seqno(0) {}
+	SplitEvaluation(int64_t epoch, int64_t seqno, Future<Void> inProgress)
+	  : epoch(epoch), seqno(seqno), inProgress(inProgress) {}
 };
 
 struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
@@ -386,7 +385,7 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 	}
 
 	if (!workerID.present()) {
-		ASSERT(assignment.isAssign);
+		ASSERT(assignment.isAssign && assignment.assign.get().type != AssignRequestType::Continue);
 		UID _workerId = wait(pickWorkerForAssign(bmData));
 		if (BM_DEBUG) {
 			fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
@@ -395,6 +394,10 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 		// We don't have to check for races with an overlapping assignment because it would insert over us in the actor
 		// map, cancelling this actor before it got here
 		bmData->workerAssignments.insert(assignment.keyRange, workerID.get());
+
+		if (bmData->workerStats.count(workerID.get())) {
+			bmData->workerStats[workerID.get()].numGranulesAssigned += 1;
+		}
 	}
 
 	if (BM_DEBUG) {
@@ -589,6 +592,11 @@ ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 				bmData->workerAssignments.insert(assignment.keyRange, workerId);
 				bmData->assignsInProgress.insert(assignment.keyRange,
 				                                 doRangeAssignment(bmData, assignment, workerId, seqNo));
+				// If we know about the worker and this is not a continue, then this is a new range for the worker
+				if (bmData->workerStats.count(workerId) &&
+				    assignment.assign.get().type != AssignRequestType::Continue) {
+					bmData->workerStats[workerId].numGranulesAssigned += 1;
+				}
 			} else {
 				// Ensure the key boundaries are updated before we pick a worker
 				bmData->workerAssignments.insert(assignment.keyRange, UID());
@@ -596,11 +604,6 @@ ACTOR Future<Void> rangeAssigner(Reference<BlobManagerData> bmData) {
 				                                 doRangeAssignment(bmData, assignment, Optional<UID>(), seqNo));
 			}
 
-			// If we know about the worker and this is not a continue, then this is a new range for the worker
-			if (bmData->workerStats.count(workerId) && assignment.assign.get().type != AssignRequestType::Continue) {
-				bmData->workerStats[workerId].numGranulesAssigned += 1;
-			}
-
 		} else {
 			if (assignment.worker.present()) {
 				// revoke this specific range from this specific worker. Either part of recovery or failing a worker
@@ -861,7 +864,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
                                    KeyRange granuleRange,
                                    UID granuleID,
                                    Version granuleStartVersion,
-                                   Version latestVersion,
                                    bool writeHot) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 	state Standalone<VectorRef<KeyRef>> newRanges;
@@ -925,11 +927,10 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 	}
 
 	if (BM_DEBUG) {
-		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3}:\n",
+		fmt::print("Splitting range [{0} - {1}) into {2} granules:\n",
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
-		           newRanges.size() - 1,
-		           latestVersion);
+		           newRanges.size() - 1);
 		for (int i = 0; i < newRanges.size(); i++) {
 			fmt::print("    {}:{}\n",
 			           (i < newGranuleIDs.size() ? newGranuleIDs[i] : UID()).toString().substr(0, 6).c_str(),
@@ -937,6 +938,8 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		}
 	}
 
+	state Version splitVersion;
+
 	// Need to split range. Persist intent to split and split metadata to DB BEFORE sending split assignments to blob
 	// workers, so that nothing is lost on blob manager recovery
 	loop {
@@ -1018,7 +1021,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			if (ownerEpoch > bmData->epoch) {
 				if (BM_DEBUG) {
 					fmt::print("BM {0} found a higher epoch {1} than {2} for granule lock of [{3} - {4})\n",
-					           bmData->id.toString(),
+					           bmData->epoch,
 					           ownerEpoch,
 					           bmData->epoch,
 					           granuleRange.begin.printable(),
@@ -1039,6 +1042,23 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			        blobGranuleLockValueFor(
 			            bmData->epoch, std::numeric_limits<int64_t>::max(), std::get<2>(prevGranuleLock)));
 
+			// get last delta file version written, to make that the split version
+			RangeResult lastDeltaFile =
+			    wait(tr->getRange(blobGranuleFileKeyRangeFor(granuleID), 1, Snapshot::False, Reverse::True));
+			ASSERT(lastDeltaFile.size() == 1);
+			std::tuple<UID, Version, uint8_t> k = decodeBlobGranuleFileKey(lastDeltaFile[0].key);
+			ASSERT(std::get<0>(k) == granuleID);
+			ASSERT(std::get<2>(k) == 'D');
+			splitVersion = std::get<1>(k);
+
+			if (BM_DEBUG) {
+				fmt::print("BM {0} found version {1} for splitting [{2} - {3})\n",
+				           bmData->epoch,
+				           splitVersion,
+				           granuleRange.begin.printable(),
+				           granuleRange.end.printable());
+			}
+
 			// set up splits in granule mapping, but point each part to the old owner (until they get reassigned)
 			state int i;
 			for (i = 0; i < newRanges.size() - 1; i++) {
@@ -1048,7 +1068,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				             blobGranuleSplitValueFor(BlobGranuleSplitState::Initialized),
 				             MutationRef::SetVersionstampedValue);
 
-				Key historyKey = blobGranuleHistoryKeyFor(KeyRangeRef(newRanges[i], newRanges[i + 1]), latestVersion);
+				Key historyKey = blobGranuleHistoryKeyFor(KeyRangeRef(newRanges[i], newRanges[i + 1]), splitVersion);
 
 				Standalone<BlobGranuleHistoryValue> historyValue;
 				historyValue.granuleID = newGranuleIDs[i];
@@ -1090,7 +1110,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
 		           newRanges.size() - 1,
-		           latestVersion);
+		           splitVersion);
 	}
 
 	// transaction committed, send range assignments
@@ -1116,7 +1136,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
 		           newRanges.size() - 1,
-		           latestVersion);
+		           splitVersion);
 	}
 
 	return Void();
@@ -1306,7 +1326,6 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 				auto lastSplitEval = bmData->splitEvaluations.rangeContaining(rep.granuleRange.begin);
 				if (rep.granuleRange.begin == lastSplitEval.begin() && rep.granuleRange.end == lastSplitEval.end() &&
 				    rep.epoch == lastSplitEval.cvalue().epoch && rep.seqno == lastSplitEval.cvalue().seqno) {
-					ASSERT(lastSplitEval.cvalue().version == rep.latestVersion);
 					if (BM_DEBUG) {
 						fmt::print("Manager {0} received repeat status for the same granule [{1} - {2}), ignoring.\n",
 						           bmData->epoch,
@@ -1349,10 +1368,9 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 						                                           rep.granuleRange,
 						                                           rep.granuleID,
 						                                           rep.startVersion,
-						                                           rep.latestVersion,
 						                                           rep.writeHotSplit);
-						bmData->splitEvaluations.insert(
-						    rep.granuleRange, SplitEvaluation(rep.epoch, rep.seqno, rep.latestVersion, doSplitEval));
+						bmData->splitEvaluations.insert(rep.granuleRange,
+						                                SplitEvaluation(rep.epoch, rep.seqno, doSplitEval));
 					}
 				}
 			}
@@ -2179,7 +2197,7 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self, UID g
 		if (latestSnapshotVersion != invalidVersion) {
 			std::string fname = files.snapshotFiles[idx].filename;
 			deletions.emplace_back(self->bstore->deleteFile(fname));
-			deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, 'S', files.snapshotFiles[idx].version));
+			deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, files.snapshotFiles[idx].version, 'S'));
 			filesToDelete.emplace_back(fname);
 		} else if (files.snapshotFiles[idx].version <= pruneVersion) {
 			// otherwise if this is the FIRST snapshot file with version < pruneVersion,
@@ -2202,7 +2220,7 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self, UID g
 		// == should also be deleted because the last delta file before a snapshot would have the same version
 		std::string fname = deltaFile.filename;
 		deletions.emplace_back(self->bstore->deleteFile(fname));
-		deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, 'D', deltaFile.version));
+		deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, deltaFile.version, 'D'));
 		filesToDelete.emplace_back(fname);
 	}
 
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 066f311163..a87d722d16 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -506,7 +506,7 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 				wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno));
 				numIterations++;
 
-				Key dfKey = blobGranuleFileKeyFor(granuleID, 'D', currentDeltaVersion);
+				Key dfKey = blobGranuleFileKeyFor(granuleID, currentDeltaVersion, 'D');
 				Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize);
 				tr->set(dfKey, dfValue);
 
@@ -643,7 +643,7 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 			try {
 				wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno));
 				numIterations++;
-				Key snapshotFileKey = blobGranuleFileKeyFor(granuleID, 'S', version);
+				Key snapshotFileKey = blobGranuleFileKeyFor(granuleID, version, 'S');
 				Key snapshotFileValue = blobGranuleFileValueFor(fname, 0, serializedSize);
 				tr->set(snapshotFileKey, snapshotFileValue);
 				// create granule history at version if this is a new granule with the initial dump from FDB
@@ -904,8 +904,7 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 				                                                                 statusEpoch,
 				                                                                 statusSeqno,
 				                                                                 granuleID,
-				                                                                 metadata->initialSnapshotVersion,
-				                                                                 reSnapshotVersion));
+				                                                                 metadata->initialSnapshotVersion));
 				break;
 			} catch (Error& e) {
 				if (e.code() == error_code_operation_cancelled) {
@@ -1900,6 +1899,12 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 					try {
 						Optional<Value> v = wait(tr.get(blobGranuleHistoryKeyFor(
 						    curHistory.value.parentGranules[0].first, curHistory.value.parentGranules[0].second)));
+						if (!v.present()) {
+							printf("No granule history present for [%s - %s) @ %lld!!\n",
+							       curHistory.value.parentGranules[0].first.begin.printable().c_str(),
+							       curHistory.value.parentGranules[0].first.end.printable().c_str(),
+							       curHistory.value.parentGranules[0].first);
+						}
 						ASSERT(v.present());
 						next = GranuleHistory(curHistory.value.parentGranules[0].first,
 						                      curHistory.value.parentGranules[0].second,

From 9d9b70abd4fc1085bea3f378bfffc320da2b83ed Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Mon, 14 Mar 2022 18:56:36 -0500
Subject: [PATCH 208/210] Fixing another BM recovery and granule splitting race

---
 fdbserver/BlobManager.actor.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 5568ede056..c48f864245 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -1550,7 +1550,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 				// new one is from DB (source of truth on boundaries) and existing mapping disagrees on boundary or
 				// assignment, do explicit revoke and re-assign to converge
 				anyConflicts = true;
-				// if ranges don't match, need to explicitly reaassign all parts of old range, as it could be from a
+				// if ranges don't match, need to explicitly reassign all parts of old range, as it could be from a
 				// yet-unassigned split
 				if (old.range() != newRange) {
 					std::get<0>(old.value()) = UID();
@@ -1564,6 +1564,11 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 			           (oldEpoch < newEpoch || (oldEpoch == newEpoch && oldSeqno < newSeqno))) {
 				// 2 blob workers reported conflicting mappings, add old one to out of date (if not already added by a
 				// previous intersecting range in the split case)
+				// if ranges don't match, need to explicitly reassign all parts of old range, as it could be from a
+				// partially-assigned split
+				if (old.range() != newRange) {
+					std::get<0>(old.value()) = UID();
+				}
 				if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) {
 					TEST(true); // BM Recovery: Two workers claim ownership of same granule
 					outOfDate.push_back(std::pair(oldWorker, old.range()));

From b27d6bbfab7244dadebb94655763b2592fdc5a02 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Tue, 15 Mar 2022 10:10:29 -0500
Subject: [PATCH 209/210] Have BW read manager lock on startup to avoid races
 with old BM

---
 fdbserver/BlobWorker.actor.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index a87d722d16..d53f5ce264 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -2872,6 +2872,12 @@ ACTOR Future<Void> registerBlobWorker(Reference<BlobWorkerData> bwData, BlobWork
 			tr->addReadConflictRange(singleKeyRange(blobWorkerListKey));
 			tr->set(blobWorkerListKey, blobWorkerListValue(interf));
 
+			// Get manager lock from DB
+			Optional<Value> currentLockValue = wait(tr->get(blobManagerEpochKey));
+			ASSERT(currentLockValue.present());
+			int64_t currentEpoch = decodeBlobManagerEpochValue(currentLockValue.get());
+			bwData->managerEpochOk(currentEpoch);
+
 			wait(tr->commit());
 
 			if (BW_DEBUG) {

From ce5b567971f830233ee42465590d7dc4d8a8f1dd Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Tue, 15 Mar 2022 14:32:25 -0500
Subject: [PATCH 210/210] Cleanup the cleanup

---
 fdbclient/DatabaseContext.h                   |  2 -
 fdbclient/NativeAPI.actor.cpp                 | 57 +------------------
 fdbclient/StorageServerInterface.h            |  5 +-
 fdbserver/BlobManager.actor.cpp               | 19 +------
 fdbserver/BlobWorker.actor.cpp                |  2 +-
 fdbserver/storageserver.actor.cpp             |  6 +-
 .../BlobGranuleCorrectnessWorkload.actor.cpp  |  3 +-
 7 files changed, 10 insertions(+), 84 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 65b47f95f9..485617c102 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -170,8 +170,6 @@ public:
 struct MutationAndVersionStream {
 	Standalone<MutationsAndVersionRef> next;
 	PromiseStream<Standalone<MutationsAndVersionRef>> results;
-	// TODO REMOVE, for debugging
-	int idx = -1;
 	bool operator<(MutationAndVersionStream const& rhs) const { return next.version > rhs.next.version; }
 };
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 07d983cd64..6207455adb 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6747,7 +6747,6 @@ Future<Standalone<VectorRef<KeyRef>>> Transaction::getRangeSplitPoints(KeyRange
 	return ::getRangeSplitPoints(trState, keys, chunkSize);
 }
 
-// TODO REMOVE when correctness clean
 #define BG_REQUEST_DEBUG false
 
 // the blob granule requests are a bit funky because they piggyback off the existing transaction to read from the system
@@ -7479,9 +7478,6 @@ Version ChangeFeedData::getVersion() {
 	return lastReturnedVersion.get();
 }
 
-// TODO REMOVE
-#define DEBUG_CF_WAIT_VERSION invalidVersion
-
 // This function is essentially bubbling the information about what has been processed from the server through the
 // change feed client. First it makes sure the server has returned all mutations up through the target version, the
 // native api has consumed and processed, them, and then the fdb client has consumed all of the mutations.
@@ -7489,28 +7485,11 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 	// wait on SS to have sent up through version
 	int desired = 0;
 	int waiting = 0;
-	if (version == DEBUG_CF_WAIT_VERSION) {
-		fmt::print("WFV {}) waitLatest\n", version);
-	}
 	std::vector<Future<Void>> allAtLeast;
 	for (auto& it : self->storageData) {
 		if (it->version.get() < version) {
 			waiting++;
-			if (version == DEBUG_CF_WAIT_VERSION) {
-				fmt::print("WFV {0})   SS {1} {2} < {3}\n",
-				           version,
-				           it->id.toString().substr(0, 4),
-				           it->version.get(),
-				           version);
-			}
 			if (version > it->desired.get()) {
-				if (version == DEBUG_CF_WAIT_VERSION) {
-					fmt::print("WFV {0})   SS {1} desired {2} -> {3}\n",
-					           version,
-					           it->id.toString().substr(0, 4),
-					           it->desired.get(),
-					           version);
-				}
 				it->desired.set(version);
 				desired++;
 			}
@@ -7518,10 +7497,6 @@ ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version
 		}
 	}
 
-	if (version == DEBUG_CF_WAIT_VERSION) {
-		fmt::print("WFV {0}) waiting for {1}/{2} ({3} desired)\n", version, waiting, self->storageData.size(), desired);
-	}
-
 	wait(waitForAll(allAtLeast));
 
 	// then, wait on ss streams to have processed up through version
@@ -7571,9 +7546,6 @@ ACTOR Future<Void> changeFeedWhenAtLatest(Reference<ChangeFeedData> self, Versio
 		return Void();
 	}
 	state Future<Void> lastReturned = self->lastReturnedVersion.whenAtLeast(version);
-	if (version == DEBUG_CF_WAIT_VERSION) {
-		fmt::print("WFV {0}) notAtLatest={1}\n", version, self->notAtLatest.get());
-	}
 	loop {
 		// only allowed to use empty versions if you're caught up
 		Future<Void> waitEmptyVersion = (self->notAtLatest.get() == 0) ? changeFeedWaitLatest(self, version) : Never();
@@ -7605,7 +7577,7 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
                                            Version end,
                                            Reference<ChangeFeedData> feedData,
                                            Reference<ChangeFeedStorageData> storageData,
-                                           UID debugUID /* TODO REMOVE */) {
+                                           UID debugUID) {
 
 	// calling lastReturnedVersion's callbacks could cause us to be cancelled
 	state Promise<Void> refresh = feedData->refresh;
@@ -7778,26 +7750,13 @@ ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> resul
 		interfNum = 0;
 		while (interfNum < streamsUsed.size()) {
 			try {
-				if (DEBUG_CF_CLIENT_TRACE) {
-					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorWait", mergeCursorUID)
-					    .detail("Idx", streamsUsed[interfNum].idx);
-				}
 				Standalone<MutationsAndVersionRef> res = waitNext(streamsUsed[interfNum].results.getFuture());
 				streamsUsed[interfNum].next = res;
 				mutations.push(streamsUsed[interfNum]);
-				if (DEBUG_CF_CLIENT_TRACE) {
-					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorGot", mergeCursorUID)
-					    .detail("Idx", streamsUsed[interfNum].idx)
-					    .detail("Version", res.version);
-				}
 			} catch (Error& e) {
 				if (e.code() != error_code_end_of_stream) {
 					throw e;
 				}
-				if (DEBUG_CF_CLIENT_TRACE) {
-					TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorEOS", mergeCursorUID)
-					    .detail("Idx", streamsUsed[interfNum].idx);
-				}
 			}
 			interfNum++;
 		}
@@ -7858,13 +7817,6 @@ ACTOR Future<Void> mergeChangeFeedStreamInternal(Reference<ChangeFeedData> resul
 	}
 }
 
-ACTOR Future<Void> onCFErrors(std::vector<Future<Void>> onErrors) {
-	wait(waitForAny(onErrors));
-	// TODO: just do waitForAny and remove onCFErrors if this doesn't trip
-	ASSERT(false);
-	return Void();
-}
-
 ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
                                          std::vector<std::pair<StorageServerInterface, KeyRange>> interfs,
                                          Reference<ChangeFeedData> results,
@@ -7880,7 +7832,6 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	TEST(interfs.size() > 10); // Large change feed merge cursor
 	TEST(interfs.size() > 100); // Very large change feed merge cursor
 
-	// TODO REMOVE eventually, useful for debugging for now
 	state UID mergeCursorUID = UID();
 	state std::vector<UID> debugUIDs;
 	results->streams.clear();
@@ -7896,7 +7847,6 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		if (replyBufferSize != -1 && req.replyBufferSize < CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES) {
 			req.replyBufferSize = CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES;
 		}
-		// TODO REMOVE
 		req.debugUID = deterministicRandom()->randomUniqueID();
 		debugUIDs.push_back(req.debugUID);
 		mergeCursorUID =
@@ -7932,7 +7882,6 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 			    .detail("End", end)
 			    .detail("CanReadPopped", canReadPopped);
 		}
-		streams[i].idx = i;
 		onErrors[i] = results->streams[i].onError();
 		fetchers[i] = partialChangeFeedStream(interfs[i].first,
 		                                      streams[i].results,
@@ -7944,7 +7893,8 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		                                      debugUIDs[i]);
 	}
 
-	wait(onCFErrors(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end, mergeCursorUID));
+	wait(waitForAny(onErrors) ||
+	     mergeChangeFeedStreamInternal(results, interfs, streams, begin, end, mergeCursorUID));
 
 	return Void();
 }
@@ -8066,7 +8016,6 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 	req.range = range;
 	req.canReadPopped = canReadPopped;
 	req.replyBufferSize = replyBufferSize;
-	// TODO REMOVE
 	req.debugUID = deterministicRandom()->randomUniqueID();
 
 	if (DEBUG_CF_CLIENT_TRACE) {
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 794397cfb2..d4d44ba7bf 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -793,8 +793,9 @@ struct ChangeFeedStreamRequest {
 	KeyRange range;
 	int replyBufferSize = -1;
 	bool canReadPopped = true;
-	// TODO REMOVE
-	UID debugUID;
+	UID debugUID; // This is only used for debugging and tracing, but being able to link a client + server side stream
+	              // is so useful for testing, and this is such small overhead compared to streaming large amounts of
+	              // change feed data, it is left in the interface
 
 	ReplyPromiseStream<ChangeFeedStreamReply> reply;
 
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index c48f864245..4934e355be 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -47,7 +47,7 @@
  * The Blob Manager is responsible for managing range granules, and recruiting and monitoring Blob Workers.
  */
 
-#define BM_DEBUG true
+#define BM_DEBUG false
 
 void handleClientBlobRange(KeyRangeMap<bool>* knownBlobRanges,
                            Arena& ar,
@@ -974,23 +974,6 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				                      existingState.size() + 2));
 				// +2 because this is boundaries and existingState was granules, and to ensure it doesn't set more
 				ASSERT(!existingBoundaries.more);
-				// TODO remove debugging printf
-				if (existingBoundaries.size() != existingState.size() + 1) {
-					printf("DBG: EB=%d, ES=%d for [%s - %s)\n",
-					       existingBoundaries.size(),
-					       existingState.size(),
-					       granuleRange.begin.printable().c_str(),
-					       granuleRange.end.printable().c_str());
-					printf("Boundaries:\n");
-					for (auto& it : existingBoundaries) {
-						printf("  %s\n", it.key.removePrefix(blobGranuleMappingKeys.begin).printable().c_str());
-					}
-					printf("State:\n");
-					for (auto& it : existingState) {
-						std::pair<UID, UID> k = decodeBlobGranuleSplitKey(it.key);
-						printf("  %s\n", k.second.toString().substr(0, 6).c_str());
-					}
-				}
 				ASSERT(existingBoundaries.size() == existingState.size() + 1);
 				newRanges.clear();
 				newRanges.arena().dependsOn(existingBoundaries.arena());
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index d53f5ce264..1e08ab3837 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -1495,9 +1495,9 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 								if (metadata->pendingDeltaVersion <= rollbackVersion &&
 								    (metadata->currentDeltas.empty() ||
 								     metadata->currentDeltas.back().version <= rollbackVersion)) {
+									TEST(true); // Granule ignoring rollback
 
 									if (BW_DEBUG) {
-										TEST(true); // Granule ignoring rollback
 										fmt::print(
 										    "Granule [{0} - {1}) on BW {2} skipping rollback {3} -> {4} completely\n",
 										    metadata->keyRange.begin.printable().c_str(),
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index a4f5b9033c..c089764b1b 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -7914,11 +7914,7 @@ ACTOR Future<Void> serveChangeFeedStreamRequests(StorageServer* self,
 	loop {
 		ChangeFeedStreamRequest req = waitNext(changeFeedStream);
 		// must notify change feed that its shard is moved away ASAP
-
-		// TODO CHANGE BACK
-		// UID streamUID = deterministicRandom()->randomUniqueID();
-		UID streamUID = req.debugUID;
-		self->actors.add(changeFeedStreamQ(self, req, streamUID) || stopChangeFeedOnMove(self, req, streamUID));
+		self->actors.add(changeFeedStreamQ(self, req, req.debugUID) || stopChangeFeedOnMove(self, req, req.debugUID));
 	}
 }
 
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index e6ec6013c7..ea43cebbb7 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -121,8 +121,7 @@ struct ThreadData : ReferenceCounted<ThreadData>, NonCopyable {
 	Key getKey(uint32_t key, uint32_t id) { return StringRef(format("%08x/%08x/%08x", directoryID, key, id)); }
 };
 
-// TODO REMOVE eventually?
-//  For debugging mismatches on what data should be and why
+// For debugging mismatches on what data should be and why
 // set mismatch to true, dir id and key id to the directory and key id that are wrong, and rv to read version that read
 // the wrong value
 #define DEBUG_MISMATCH false