Merge branch 'main' of https://github.com/apple/foundationdb into features/debug-macro

bindings/c/CMakeLists.txt

@@ -306,7 +306,7 @@ endif()
       COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
             --build-dir ${CMAKE_BINARY_DIR}
             --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-            --upgrade-path "6.3.23" "7.0.0" "7.2.0"
+            --upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0"
             --process-number 1
             )
 
@@ -314,7 +314,7 @@ endif()
       COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
             --build-dir ${CMAKE_BINARY_DIR}
             --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-            --upgrade-path "7.0.0" "7.2.0"
+            --upgrade-path "7.0.0" "7.1.5" "7.2.0"
             --process-number 1
             )
 
@@ -322,7 +322,7 @@ endif()
         COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
             --build-dir ${CMAKE_BINARY_DIR}
             --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-            --upgrade-path "6.3.23" "7.0.0" "7.2.0"
+            --upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0" "7.1.5"
             --process-number 3
           )
 
@@ -330,9 +330,38 @@ endif()
         COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
             --build-dir ${CMAKE_BINARY_DIR}
             --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-            --upgrade-path "7.0.0" "7.2.0"
+            --upgrade-path "7.0.0" "7.1.5" "7.2.0" "7.1.5"
             --process-number 3
           )
+
+    add_test(NAME fdb_c_upgrade_multi_threaded_710api
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+            --upgrade-path "7.1.5" "7.2.0" "7.1.5"
+            --process-number 3
+          )
+
+    add_test(NAME fdb_c_cluster_wiggle
+          COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+              --build-dir ${CMAKE_BINARY_DIR}
+              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+              --upgrade-path "7.2.0" "wiggle"
+              --disable-log-dump
+              --process-number 3
+              --redundancy double
+            )
+
+    add_test(NAME fdb_c_wiggle_and_upgrade
+          COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+              --build-dir ${CMAKE_BINARY_DIR}
+              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+              --upgrade-path "7.0.0" "wiggle" "7.2.0"
+              --disable-log-dump
+              --process-number 3
+              --redundancy double
+            )
+
   endif()
 
 endif()

bindings/c/fdb_c.cpp

@@ -655,6 +655,7 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_mapped_range(FDBTransaction*
                                                                  int target_bytes,
                                                                  FDBStreamingMode mode,
                                                                  int iteration,
+                                                                 int matchIndex,
                                                                  fdb_bool_t snapshot,
                                                                  fdb_bool_t reverse) {
 	FDBFuture* r = validate_and_update_parameters(limit, target_bytes, mode, iteration, reverse);
@@ -667,6 +668,7 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_mapped_range(FDBTransaction*
 	                        KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset),
 	                        StringRef(mapper_name, mapper_name_length),
 	                        GetRangeLimits(limit, target_bytes),
+	                        matchIndex,
 	                        snapshot,
 	                        reverse)
 	                    .extractPtr());

bindings/c/foundationdb/fdb_c.h

@@ -160,6 +160,7 @@ typedef struct mappedkeyvalue {
 	 * take the shortcut. */
 	FDBGetRangeReqAndResult getRange;
 	unsigned char buffer[32];
+	fdb_bool_t boundaryAndExist;
 } FDBMappedKeyValue;
 
 #pragma pack(push, 4)
@@ -384,6 +385,7 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_mapped_range(FDBTran
                                                                          int target_bytes,
                                                                          FDBStreamingMode mode,
                                                                          int iteration,
+                                                                         int matchIndex,
                                                                          fdb_bool_t snapshot,
                                                                          fdb_bool_t reverse);
 

bindings/c/test/apitester/TesterOptions.h

@@ -48,6 +48,7 @@ public:
 	int numClientThreads;
 	int numDatabases;
 	int numClients;
+	int statsIntervalMs = 0;
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
 	std::string bgBasePath;

bindings/c/test/apitester/TesterScheduler.cpp

@@ -21,6 +21,7 @@
 #include "TesterScheduler.h"
 #include "TesterUtil.h"
 
+#include <boost/asio/detail/chrono.hpp>
 #include <memory>
 #include <thread>
 #include <boost/asio.hpp>
@@ -31,6 +32,15 @@ namespace FdbApiTester {
 
 const TTaskFct NO_OP_TASK = []() {};
 
+class AsioTimer : public ITimer {
+public:
+	AsioTimer(io_context& io_ctx, chrono::steady_clock::duration time) : impl(io_ctx, time) {}
+
+	void cancel() override { impl.cancel(); }
+
+	boost::asio::steady_timer impl;
+};
+
 class AsioScheduler : public IScheduler {
 public:
 	AsioScheduler(int numThreads) : numThreads(numThreads) {}
@@ -44,6 +54,16 @@ public:
 
 	void schedule(TTaskFct task) override { post(io_ctx, task); }
 
+	std::unique_ptr<ITimer> scheduleWithDelay(int delayMs, TTaskFct task) override {
+		auto timer = std::make_unique<AsioTimer>(io_ctx, boost::asio::chrono::milliseconds(delayMs));
+		timer->impl.async_wait([task](const boost::system::error_code& e) {
+			if (!e) {
+				task();
+			}
+		});
+		return timer;
+	}
+
 	void stop() override { work = any_io_executor(); }
 
 	void join() override {

bindings/c/test/apitester/TesterScheduler.h

@@ -32,6 +32,16 @@ using TTaskFct = std::function<void(void)>;
 
 extern const TTaskFct NO_OP_TASK;
 
+/**
+ * Handle to a scheduled timer
+ */
+class ITimer {
+public:
+	virtual ~ITimer() {}
+
+	virtual void cancel() = 0;
+};
+
 /**
  * Scheduler for asynchronous execution of tasks on a pool of threads
  */
@@ -45,6 +55,9 @@ public:
 	// Schedule a task for asynchronous execution
 	virtual void schedule(TTaskFct task) = 0;
 
+	// Schedule a task to be executed with a given delay
+	virtual std::unique_ptr<ITimer> scheduleWithDelay(int delayMs, TTaskFct task) = 0;
+
 	// Gracefully stop the scheduler. Waits for already running tasks to be finish
 	virtual void stop() = 0;
 

bindings/c/test/apitester/TesterTransactionExecutor.cpp

@@ -33,8 +33,8 @@
 
 namespace FdbApiTester {
 
-constexpr int LONG_WAIT_TIME_US = 1000000;
+constexpr int LONG_WAIT_TIME_US = 2000000;
-constexpr int LARGE_NUMBER_OF_RETRIES = 5;
+constexpr int LARGE_NUMBER_OF_RETRIES = 10;
 
 void TransactionActorBase::complete(fdb_error_t err) {
 	error = err;

bindings/c/test/apitester/TesterWorkload.cpp

@@ -80,7 +80,7 @@ bool WorkloadConfig::getBoolOption(const std::string& name, bool defaultVal) con
 
 WorkloadBase::WorkloadBase(const WorkloadConfig& config)
   : manager(nullptr), tasksScheduled(0), numErrors(0), clientId(config.clientId), numClients(config.numClients),
-    failed(false) {
+    failed(false), numTxCompleted(0) {
 	maxErrors = config.getIntOption("maxErrors", 10);
 	workloadId = fmt::format("{}{}", config.name, clientId);
 }
@@ -89,6 +89,10 @@ void WorkloadBase::init(WorkloadManager* manager) {
 	this->manager = manager;
 }
 
+void WorkloadBase::printStats() {
+	info(fmt::format("{} transactions completed", numTxCompleted.load()));
+}
+
 void WorkloadBase::schedule(TTaskFct task) {
 	if (failed) {
 		return;
@@ -106,6 +110,7 @@ void WorkloadBase::execTransaction(std::shared_ptr<ITransactionActor> tx, TTaskF
 	}
 	tasksScheduled++;
 	manager->txExecutor->execute(tx, [this, tx, cont, failOnError]() {
+		numTxCompleted++;
 		fdb_error_t err = tx->getErrorCode();
 		if (tx->getErrorCode() == error_code_success) {
 			cont();
@@ -198,6 +203,9 @@ void WorkloadManager::workloadDone(IWorkload* workload, bool failed) {
 	bool done = workloads.empty();
 	lock.unlock();
 	if (done) {
+		if (statsTimer) {
+			statsTimer->cancel();
+		}
 		scheduler->stop();
 	}
 }
@@ -241,6 +249,24 @@ void WorkloadManager::readControlInput(std::string pipeName) {
 	}
 }
 
+void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
+	statsTimer = scheduler->scheduleWithDelay(timeIntervalMs, [this, timeIntervalMs]() {
+		for (auto workload : getActiveWorkloads()) {
+			workload->printStats();
+		}
+		this->schedulePrintStatistics(timeIntervalMs);
+	});
+}
+
+std::vector<std::shared_ptr<IWorkload>> WorkloadManager::getActiveWorkloads() {
+	std::unique_lock<std::mutex> lock(mutex);
+	std::vector<std::shared_ptr<IWorkload>> res;
+	for (auto iter : workloads) {
+		res.push_back(iter.second.ref);
+	}
+	return res;
+}
+
 void WorkloadManager::handleStopCommand() {
 	std::unique_lock<std::mutex> lock(mutex);
 	for (auto& iter : workloads) {

bindings/c/test/apitester/TesterWorkload.h

@@ -62,6 +62,9 @@ public:
 
 	// Get workload control interface if supported, nullptr otherwise
 	virtual IWorkloadControlIfc* getControlIfc() = 0;
+
+	// Print workload statistics
+	virtual void printStats() = 0;
 };
 
 // Workload configuration
@@ -100,6 +103,8 @@ public:
 
 	std::string getWorkloadId() override { return workloadId; }
 
+	void printStats() override;
+
 protected:
 	// Schedule the a task as a part of the workload
 	void schedule(TTaskFct task);
@@ -150,6 +155,9 @@ protected:
 
 	// Workload is failed, no further transactions or continuations will be scheduled by the workload
 	std::atomic<bool> failed;
+
+	// Number of completed transactions
+	std::atomic<int> numTxCompleted;
 };
 
 // Workload manager
@@ -175,6 +183,9 @@ public:
 		return numWorkloadsFailed > 0;
 	}
 
+	// Schedule statistics to be printed in regular timeintervals
+	void schedulePrintStatistics(int timeIntervalMs);
+
 private:
 	friend WorkloadBase;
 
@@ -205,6 +216,9 @@ private:
 	// Handle CHECK command received from the test controller
 	void handleCheckCommand();
 
+	// A thread-safe operation to return a list of active workloads
+	std::vector<std::shared_ptr<IWorkload>> getActiveWorkloads();
+
 	// Transaction executor to be used by the workloads
 	ITransactionExecutor* txExecutor;
 
@@ -225,6 +239,9 @@ private:
 
 	// Output pipe for emitting test control events
 	std::ofstream outputPipe;
+
+	// Timer for printing statistics in regular intervals
+	std::unique_ptr<ITimer> statsTimer;
 };
 
 // A workload factory

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -53,7 +53,8 @@ enum TesterOptionId {
 	OPT_OUTPUT_PIPE,
 	OPT_FDB_API_VERSION,
 	OPT_TRANSACTION_RETRY_LIMIT,
-	OPT_BLOB_GRANULE_LOCAL_FILE_PATH
+	OPT_BLOB_GRANULE_LOCAL_FILE_PATH,
+	OPT_STATS_INTERVAL
 };
 
 CSimpleOpt::SOption TesterOptionDefs[] = //
@@ -77,6 +78,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_FDB_API_VERSION, "--api-version", SO_REQ_SEP },
 	  { OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP },
 	  { OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP },
+	  { OPT_STATS_INTERVAL, "--stats-interval", SO_REQ_SEP },
 	  SO_END_OF_OPTIONS };
 
 void printProgramUsage(const char* execName) {
@@ -118,6 +120,8 @@ void printProgramUsage(const char* execName) {
 	       "				 Path to blob granule files on local filesystem\n"
 	       "  -f, --test-file FILE\n"
 	       "                 Test file to run.\n"
+	       "  --stats-interval MILLISECONDS\n"
+	       "                 Time interval in milliseconds for printing workload statistics (default: 0 - disabled).\n"
 	       "  -h, --help     Display this help and exit.\n",
 	       FDB_API_VERSION);
 }
@@ -214,6 +218,9 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 	case OPT_BLOB_GRANULE_LOCAL_FILE_PATH:
 		options.bgBasePath = args.OptionArg();
 		break;
+	case OPT_STATS_INTERVAL:
+		processIntOption(args.OptionText(), args.OptionArg(), 0, 60000, options.statsIntervalMs);
+		break;
 	}
 	return true;
 }
@@ -335,6 +342,9 @@ bool runWorkloads(TesterOptions& options) {
 		}
 
 		scheduler->start();
+		if (options.statsIntervalMs) {
+			workloadMgr.schedulePrintStatistics(options.statsIntervalMs);
+		}
 		workloadMgr.run();
 		return !workloadMgr.failed();
 	} catch (const std::runtime_error& err) {

bindings/c/test/apitester/run_c_api_tests.py

@@ -30,6 +30,8 @@ import glob
 import random
 import string
 
+TESTER_STATS_INTERVAL_SEC = 5
+
 
 def random_string(len):
     return ''.join(random.choice(string.ascii_letters + string.digits) for i in range(len))
@@ -66,7 +68,8 @@ def dump_client_logs(log_dir):
 def run_tester(args, test_file):
     cmd = [args.tester_binary,
            "--cluster-file", args.cluster_file,
-           "--test-file", test_file]
+           "--test-file", test_file,
+           "--stats-interval", str(TESTER_STATS_INTERVAL_SEC*1000)]
     if args.external_client_library is not None:
         cmd += ["--external-client-library", args.external_client_library]
     if args.tmp_dir is not None:

bindings/c/test/mako/mako.cpp

@@ -619,6 +619,19 @@ int workerProcessMain(Arguments const& args, int worker_id, shared_memory::Acces
 
 	selectApiVersion(args.api_version);
 
+	/* enable distributed tracing */
+	switch (args.distributed_tracer_client) {
+	case 1:
+		err = network::setOptionNothrow(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER, BytesRef(toBytePtr("network_lossy")));
+		break;
+	case 2:
+		err = network::setOptionNothrow(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER, BytesRef(toBytePtr("log_file")));
+		break;
+	}
+	if (err) {
+		logr.error("network::setOption(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER): {}", err.what());
+	}
+
 	/* enable flatbuffers if specified */
 	if (args.flatbuffers) {
 #ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@@ -824,6 +837,7 @@ int initArguments(Arguments& args) {
 	args.json_output_path[0] = '\0';
 	args.bg_materialize_files = false;
 	args.bg_file_path[0] = '\0';
+	args.distributed_tracer_client = 0;
 	return 0;
 }
 
@@ -1002,6 +1016,8 @@ void usage() {
 	printf("%-24s %s\n",
 	       "    --bg_file_path=PATH",
 	       "Read blob granule files from the local filesystem at PATH and materialize the results.");
+	printf(
+	    "%-24s %s\n", "    --distributed_tracer_client=CLIENT", "Specify client (disabled, network_lossy, log_file)");
 }
 
 /* parse benchmark paramters */
@@ -1053,6 +1069,7 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 			{ "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW },
 			{ "json_report", optional_argument, NULL, ARG_JSON_REPORT },
 			{ "bg_file_path", required_argument, NULL, ARG_BG_FILE_PATH },
+			{ "distributed_tracer_client", required_argument, NULL, ARG_DISTRIBUTED_TRACER_CLIENT },
 			{ NULL, 0, NULL, 0 }
 		};
 		idx = 0;
@@ -1240,6 +1257,17 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 		case ARG_BG_FILE_PATH:
 			args.bg_materialize_files = true;
 			strncpy(args.bg_file_path, optarg, std::min(sizeof(args.bg_file_path), strlen(optarg) + 1));
+		case ARG_DISTRIBUTED_TRACER_CLIENT:
+			if (strcmp(optarg, "disabled") == 0) {
+				args.distributed_tracer_client = 0;
+			} else if (strcmp(optarg, "network_lossy") == 0) {
+				args.distributed_tracer_client = 1;
+			} else if (strcmp(optarg, "log_file") == 0) {
+				args.distributed_tracer_client = 2;
+			} else {
+				args.distributed_tracer_client = -1;
+			}
+			break;
 		}
 	}
 
@@ -1307,6 +1335,10 @@ int validateArguments(Arguments const& args) {
 			return -1;
 		}
 	}
+	if (args.distributed_tracer_client < 0) {
+		logr.error("--disibuted_tracer_client must specify either (disabled, network_lossy, log_file)");
+		return -1;
+	}
 	return 0;
 }
 

bindings/c/test/mako/mako.hpp

@@ -72,7 +72,8 @@ enum ArgKind {
 	ARG_DISABLE_RYW,
 	ARG_CLIENT_THREADS_PER_VERSION,
 	ARG_JSON_REPORT,
-	ARG_BG_FILE_PATH // if blob granule files are stored locally, mako will read and materialize them if this is set
+	ARG_BG_FILE_PATH, // if blob granule files are stored locally, mako will read and materialize them if this is set
+	ARG_DISTRIBUTED_TRACER_CLIENT
 };
 
 constexpr const int OP_COUNT = 0;
@@ -161,6 +162,7 @@ struct Arguments {
 	char json_output_path[PATH_MAX];
 	bool bg_materialize_files;
 	char bg_file_path[PATH_MAX];
+	int distributed_tracer_client;
 };
 
 } // namespace mako

bindings/c/test/unit/fdb_api.cpp

@@ -271,6 +271,7 @@ MappedKeyValueArrayFuture Transaction::get_mapped_range(const uint8_t* begin_key
                                                         int target_bytes,
                                                         FDBStreamingMode mode,
                                                         int iteration,
+                                                        int matchIndex,
                                                         fdb_bool_t snapshot,
                                                         fdb_bool_t reverse) {
 	return MappedKeyValueArrayFuture(fdb_transaction_get_mapped_range(tr_,
@@ -288,6 +289,7 @@ MappedKeyValueArrayFuture Transaction::get_mapped_range(const uint8_t* begin_key
 	                                                                  target_bytes,
 	                                                                  mode,
 	                                                                  iteration,
+	                                                                  matchIndex,
 	                                                                  snapshot,
 	                                                                  reverse));
 }

bindings/c/test/unit/fdb_api.hpp

@@ -304,6 +304,7 @@ public:
 	                                           int target_bytes,
 	                                           FDBStreamingMode mode,
 	                                           int iteration,
+	                                           int matchIndex,
 	                                           fdb_bool_t snapshot,
 	                                           fdb_bool_t reverse);
 

bindings/c/test/unit/unit_tests.cpp

@@ -181,8 +181,8 @@ struct GetMappedRangeResult {
 	                       std::string, // value
 	                       std::string, // begin
 	                       std::string, // end
-	                       std::vector<std::pair<std::string, std::string>> // range results
+	                       std::vector<std::pair<std::string, std::string>>, // range results
-	                       >>
+	                       fdb_bool_t>>
 	    mkvs;
 	// True if values remain in the key range requested.
 	bool more;
@@ -261,6 +261,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
                                       int target_bytes,
                                       FDBStreamingMode mode,
                                       int iteration,
+                                      int matchIndex,
                                       fdb_bool_t snapshot,
                                       fdb_bool_t reverse) {
 	fdb::MappedKeyValueArrayFuture f1 = tr.get_mapped_range(begin_key_name,
@@ -277,6 +278,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
 	                                                        target_bytes,
 	                                                        mode,
 	                                                        iteration,
+	                                                        matchIndex,
 	                                                        snapshot,
 	                                                        reverse);
 
@@ -304,6 +306,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
 		auto value = extractString(mkv.value);
 		auto begin = extractString(mkv.getRange.begin.key);
 		auto end = extractString(mkv.getRange.end.key);
+		bool boundaryAndExist = mkv.boundaryAndExist;
 		//		std::cout << "key:" << key << " value:" << value << " begin:" << begin << " end:" << end << std::endl;
 
 		std::vector<std::pair<std::string, std::string>> range_results;
@@ -314,7 +317,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
 			range_results.emplace_back(k, v);
 			// std::cout << "[" << i << "]" << k << " -> " << v << std::endl;
 		}
-		result.mkvs.emplace_back(key, value, begin, end, range_results);
+		result.mkvs.emplace_back(key, value, begin, end, range_results, boundaryAndExist);
 	}
 	return result;
 }
@@ -951,7 +954,11 @@ std::map<std::string, std::string> fillInRecords(int n) {
 	return data;
 }
 
-GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr, std::string mapper) {
+GetMappedRangeResult getMappedIndexEntries(int beginId,
+                                           int endId,
+                                           fdb::Transaction& tr,
+                                           std::string mapper,
+                                           int matchIndex) {
 	std::string indexEntryKeyBegin = indexEntryKey(beginId);
 	std::string indexEntryKeyEnd = indexEntryKey(endId);
 
@@ -965,13 +972,24 @@ GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transact
 	    /* target_bytes */ 0,
 	    /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
 	    /* iteration */ 0,
+	    /* matchIndex */ matchIndex,
 	    /* snapshot */ false,
 	    /* reverse */ 0);
 }
 
-GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr) {
+GetMappedRangeResult getMappedIndexEntries(int beginId,
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString();
+                                           int endId,
-	return getMappedIndexEntries(beginId, endId, tr, mapper);
+                                           fdb::Transaction& tr,
+                                           int matchIndex,
+                                           bool allMissing) {
+	std::string mapper = Tuple()
+	                         .append(prefix)
+	                         .append(RECORD)
+	                         .append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
+	                         .append("{...}"_sr)
+	                         .pack()
+	                         .toString();
+	return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
 }
 
 TEST_CASE("fdb_transaction_get_mapped_range") {
@@ -983,7 +1001,16 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 	while (1) {
 		int beginId = 1;
 		int endId = 19;
-		auto result = getMappedIndexEntries(beginId, endId, tr);
+		const double r = deterministicRandom()->random01();
+		int matchIndex = MATCH_INDEX_ALL;
+		if (r < 0.25) {
+			matchIndex = MATCH_INDEX_NONE;
+		} else if (r < 0.5) {
+			matchIndex = MATCH_INDEX_MATCHED_ONLY;
+		} else if (r < 0.75) {
+			matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
+		}
+		auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, false);
 
 		if (result.err) {
 			fdb::EmptyFuture f1 = tr.on_error(result.err);
@@ -996,9 +1023,21 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 		CHECK(!result.more);
 
 		int id = beginId;
+		bool boundary;
 		for (int i = 0; i < expectSize; i++, id++) {
-			const auto& [key, value, begin, end, range_results] = result.mkvs[i];
+			boundary = i == 0 || i == expectSize - 1;
-			CHECK(indexEntryKey(id).compare(key) == 0);
+			const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
+			if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
+				CHECK(EMPTY.compare(key) == 0);
+			} else {
+				CHECK(EMPTY.compare(key) == 0);
+			}
+			bool empty = range_results.empty();
+			CHECK(boundaryAndExist == (boundary && !empty));
 			CHECK(EMPTY.compare(value) == 0);
 			CHECK(range_results.size() == SPLIT_SIZE);
 			for (int split = 0; split < SPLIT_SIZE; split++) {
@@ -1011,6 +1050,58 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 	}
 }
 
+TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
+	const int TOTAL_RECORDS = 20;
+	fillInRecords(TOTAL_RECORDS);
+
+	fdb::Transaction tr(db);
+	// RYW should be enabled.
+	while (1) {
+		int beginId = 1;
+		int endId = 19;
+		const double r = deterministicRandom()->random01();
+		int matchIndex = MATCH_INDEX_ALL;
+		if (r < 0.25) {
+			matchIndex = MATCH_INDEX_NONE;
+		} else if (r < 0.5) {
+			matchIndex = MATCH_INDEX_MATCHED_ONLY;
+		} else if (r < 0.75) {
+			matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
+		}
+		auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, true);
+
+		if (result.err) {
+			fdb::EmptyFuture f1 = tr.on_error(result.err);
+			fdb_check(wait_future(f1));
+			continue;
+		}
+
+		int expectSize = endId - beginId;
+		CHECK(result.mkvs.size() == expectSize);
+		CHECK(!result.more);
+
+		int id = beginId;
+		bool boundary;
+		for (int i = 0; i < expectSize; i++, id++) {
+			boundary = i == 0 || i == expectSize - 1;
+			const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
+			if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
+				CHECK(EMPTY.compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else {
+				CHECK(EMPTY.compare(key) == 0);
+			}
+			bool empty = range_results.empty();
+			CHECK(boundaryAndExist == (boundary && !empty));
+			CHECK(EMPTY.compare(value) == 0);
+		}
+		break;
+	}
+}
+
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
 	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
@@ -1024,6 +1115,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
 	    /* target_bytes */ 0,
 	    /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
 	    /* iteration */ 0,
+	    /* matchIndex */ MATCH_INDEX_ALL,
 	    /* snapshot */ true, // Set snapshot to true
 	    /* reverse */ 0);
 	ASSERT(result.err == error_code_unsupported_operation);
@@ -1043,6 +1135,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_ryw_enable") {
 	    /* target_bytes */ 0,
 	    /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
 	    /* iteration */ 0,
+	    /* matchIndex */ MATCH_INDEX_ALL,
 	    /* snapshot */ false,
 	    /* reverse */ 0);
 	ASSERT(result.err == error_code_unsupported_operation);
@@ -1069,7 +1162,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") {
 	};
 	assertNotTuple(mapper);
 	fdb::Transaction tr(db);
-	auto result = getMappedIndexEntries(1, 3, tr, mapper);
+	auto result = getMappedIndexEntries(1, 3, tr, mapper, MATCH_INDEX_ALL);
 	ASSERT(result.err == error_code_mapper_not_tuple);
 }
 

bindings/go/src/fdb/generated.go

@@ -289,7 +289,7 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error {
 //
 // Parameter: Client directory for temporary files. 
 func (o NetworkOptions) SetClientTmpDir(param string) error {
-	return o.setOpt(90, []byte(param))
+	return o.setOpt(91, []byte(param))
 }
 
 // Set the size of the client location cache. Raising this value can boost performance in very large databases where clients access data in a near-random pattern. Defaults to 100000.

bindings/java/fdbJNI.cpp

@@ -533,10 +533,14 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 		FDBMappedKeyValue kvm = kvms[i];
 		int kvm_count = kvm.getRange.m_size;
 
-		const int totalLengths = 4 + kvm_count * 2;
+		// now it has 5 field, key, value, getRange.begin, getRange.end, boundaryAndExist
+		// this needs to change if FDBMappedKeyValue definition is changed.
+		const int totalFieldFDBMappedKeyValue = 5;
+
+		const int totalLengths = totalFieldFDBMappedKeyValue + kvm_count * 2;
 
 		int totalBytes = kvm.key.key_length + kvm.value.key_length + kvm.getRange.begin.key.key_length +
-		                 kvm.getRange.end.key.key_length;
+		                 kvm.getRange.end.key.key_length + sizeof(kvm.boundaryAndExist);
 		for (int i = 0; i < kvm_count; i++) {
 			auto kv = kvm.getRange.data[i];
 			totalBytes += kv.key_length + kv.value_length;
@@ -580,6 +584,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 				cpBytesAndLength(pByte, pLength, kvm.value);
 				cpBytesAndLength(pByte, pLength, kvm.getRange.begin.key);
 				cpBytesAndLength(pByte, pLength, kvm.getRange.end.key);
+				cpBytesAndLengthInner(pByte, pLength, (uint8_t*)&(kvm.boundaryAndExist), sizeof(kvm.boundaryAndExist));
 				for (int kvm_i = 0; kvm_i < kvm_count; kvm_i++) {
 					auto kv = kvm.getRange.data[kvm_i];
 					cpBytesAndLengthInner(pByte, pLength, kv.key, kv.key_length);
@@ -588,6 +593,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 			}
 		}
 		// After native arrays are released
+		// call public static method MappedKeyValue::fromBytes()
 		jobject mkv = jenv->CallStaticObjectMethod(
 		    mapped_key_value_class, mapped_key_value_from_bytes, (jbyteArray)bytesArray, (jintArray)lengthArray);
 		if (jenv->ExceptionOccurred())
@@ -960,6 +966,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
                                                                                                jint targetBytes,
                                                                                                jint streamingMode,
                                                                                                jint iteration,
+                                                                                               jint matchIndex,
                                                                                                jboolean snapshot,
                                                                                                jboolean reverse) {
 	if (!tPtr || !keyBeginBytes || !keyEndBytes || !mapperBytes) {
@@ -1007,6 +1014,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 	                                                targetBytes,
 	                                                (FDBStreamingMode)streamingMode,
 	                                                iteration,
+	                                                matchIndex,
 	                                                snapshot,
 	                                                reverse);
 	jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);

bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java

@@ -43,8 +43,8 @@ public class CycleMultiClientIntegrationTest {
     public static final MultiClientHelper clientHelper = new MultiClientHelper();
 
     // more write txn than validate txn, as parent thread waits only for validate txn.
-    private static final int writeTxnCnt = 2000;
+    private static final int writeTxnCnt = 200;
-    private static final int validateTxnCnt = 1000;
+    private static final int validateTxnCnt = 100;
     private static final int threadPerDB = 5;
 
     private static final int cycleLength = 4;

bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java

@@ -192,12 +192,12 @@ class MappedRangeQueryIntegrationTest {
 
 	RangeQueryWithIndex mappedRangeQuery = (int begin, int end, Database db) -> db.run(tr -> {
 		try {
-			List<MappedKeyValue> kvs =
+			List<MappedKeyValue> kvs = tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
-			    tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
+			                                             KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
-			                          KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
+			                                             ReadTransaction.ROW_LIMIT_UNLIMITED,
-			                          ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
+			                                             FDBTransaction.MATCH_INDEX_ALL, false, StreamingMode.WANT_ALL)
-			        .asList()
+			                               .asList()
-			        .get();
+			                               .get();
 			Assertions.assertEquals(end - begin, kvs.size());
 
 			if (validate) {
@@ -208,7 +208,11 @@ class MappedRangeQueryIntegrationTest {
 					assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
 					assertByteArrayEquals(EMPTY, mappedKeyValue.getValue());
 					assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
-
+					if (id == begin || id == end - 1) {
+						Assertions.assertTrue(mappedKeyValue.getBoundaryAndExist());
+					} else {
+						Assertions.assertFalse(mappedKeyValue.getBoundaryAndExist());
+					}
 					byte[] prefix = recordKeyPrefix(id);
 					assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin());
 					prefix[prefix.length - 1] = (byte)0x01;

bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java

@@ -32,6 +32,12 @@ import com.apple.foundationdb.async.AsyncUtil;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 
 class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionConsumer {
+
+	static public final int MATCH_INDEX_ALL = 0;
+	static public final int MATCH_INDEX_NONE = 1;
+	static public final int MATCH_INDEX_MATCHED_ONLY = 2;
+	static public final int MATCH_INDEX_UNMATCHED_ONLY = 3;
+
 	private final Database database;
 	private final Executor executor;
 	private final TransactionOptions options;
@@ -93,7 +99,8 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 
 		@Override
 		public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper,
-		                                                        int limit, boolean reverse, StreamingMode mode) {
+		                                                    int limit, int matchIndex, boolean reverse,
+		                                                    StreamingMode mode) {
 
 			throw new UnsupportedOperationException("getMappedRange is only supported in serializable");
 		}
@@ -346,12 +353,13 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 	}
 
 	@Override
-	public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper,
+	public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper, int limit,
-	                                                        int limit, boolean reverse, StreamingMode mode) {
+	                                                    int matchIndex, boolean reverse, StreamingMode mode) {
 		if (mapper == null) {
 			throw new IllegalArgumentException("Mapper must be non-null");
 		}
-		return new MappedRangeQuery(FDBTransaction.this, false, begin, end, mapper, limit, reverse, mode, eventKeeper);
+		return new MappedRangeQuery(FDBTransaction.this, false, begin, end, mapper, limit, matchIndex, reverse, mode,
+		                            eventKeeper);
 	}
 
 	///////////////////
@@ -456,7 +464,8 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 	protected FutureMappedResults getMappedRange_internal(KeySelector begin, KeySelector end,
 	                                                      byte[] mapper, // Nullable
 	                                                      int rowLimit, int targetBytes, int streamingMode,
-	                                                      int iteration, boolean isSnapshot, boolean reverse) {
+	                                                      int iteration, boolean isSnapshot, boolean reverse,
+	                                                      int matchIndex) {
 		if (eventKeeper != null) {
 			eventKeeper.increment(Events.JNI_CALL);
 		}
@@ -467,9 +476,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 			    begin.toString(), end.toString(), rowLimit, targetBytes, streamingMode,
 			    iteration, Boolean.toString(isSnapshot), Boolean.toString(reverse)));*/
 			return new FutureMappedResults(
-			    Transaction_getMappedRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(),
+			    Transaction_getMappedRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(), end.getKey(),
-			                                   end.getKey(), end.orEqual(), end.getOffset(), mapper, rowLimit,
+			                               end.orEqual(), end.getOffset(), mapper, rowLimit, targetBytes, streamingMode,
-			                                   targetBytes, streamingMode, iteration, isSnapshot, reverse),
+			                               iteration, matchIndex, isSnapshot, reverse),
 			    FDB.instance().isDirectBufferQueriesEnabled(), executor, eventKeeper);
 		} finally {
 			pointerReadLock.unlock();
@@ -809,12 +818,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 			byte[] keyEnd, boolean orEqualEnd, int offsetEnd,
 			int rowLimit, int targetBytes, int streamingMode, int iteration,
 			boolean isSnapshot, boolean reverse);
-	private native long Transaction_getMappedRange(long cPtr, byte[] keyBegin, boolean orEqualBegin,
+	private native long Transaction_getMappedRange(long cPtr, byte[] keyBegin, boolean orEqualBegin, int offsetBegin,
-	                                                   int offsetBegin, byte[] keyEnd, boolean orEqualEnd,
+	                                               byte[] keyEnd, boolean orEqualEnd, int offsetEnd,
-	                                                   int offsetEnd,
+	                                               byte[] mapper, // Nonnull
-	                                                   byte[] mapper, // Nonnull
+	                                               int rowLimit, int targetBytes, int streamingMode, int iteration,
-	                                                   int rowLimit, int targetBytes, int streamingMode, int iteration,
+	                                               int matchIndex, boolean isSnapshot, boolean reverse);
-	                                                   boolean isSnapshot, boolean reverse);
 	private native void Transaction_addConflictRange(long cPtr,
 			byte[] keyBegin, byte[] keyEnd, int conflictRangeType);
 	private native void Transaction_set(long cPtr, byte[] key, byte[] value);

bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java

@@ -41,4 +41,8 @@ public class KeyArrayResult {
 			keys.add(key);
 		}
 	}
+	
+	public List<byte[]> getKeys() {
+		return keys;	
+	}
 }

bindings/java/src/main/com/apple/foundationdb/MappedKeyValue.java

@@ -22,6 +22,8 @@ package com.apple.foundationdb;
 
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -31,24 +33,35 @@ public class MappedKeyValue extends KeyValue {
 	private final byte[] rangeBegin;
 	private final byte[] rangeEnd;
 	private final List<KeyValue> rangeResult;
+	private final int boundaryAndExist;
 
-	MappedKeyValue(byte[] key, byte[] value, byte[] rangeBegin, byte[] rangeEnd, List<KeyValue> rangeResult) {
+	// now it has 5 field, key, value, getRange.begin, getRange.end, boundaryAndExist
+	// this needs to change if FDBMappedKeyValue definition is changed.
+	private static final int TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue = 5;
+
+	public MappedKeyValue(byte[] key, byte[] value, byte[] rangeBegin, byte[] rangeEnd, List<KeyValue> rangeResult,
+	               int boundaryAndExist) {
 		super(key, value);
 		this.rangeBegin = rangeBegin;
 		this.rangeEnd = rangeEnd;
 		this.rangeResult = rangeResult;
+		this.boundaryAndExist = boundaryAndExist;
 	}
 
 	public byte[] getRangeBegin() { return rangeBegin; }
 
 	public byte[] getRangeEnd() { return rangeEnd; }
 
+	public boolean getBoundaryAndExist() { return boundaryAndExist == 0 ? false : true; }
+
 	public List<KeyValue> getRangeResult() { return rangeResult; }
 
 	public static MappedKeyValue fromBytes(byte[] bytes, int[] lengths) {
 		// Lengths include: key, value, rangeBegin, rangeEnd, count * (underlying key, underlying value)
-		if (lengths.length < 4) {
+		if (lengths.length < TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) {
-			throw new IllegalArgumentException("There needs to be at least 4 lengths to cover the metadata");
+			throw new IllegalArgumentException("There needs to be at least " +
+			                                   TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue +
+			                                   " lengths to cover the metadata");
 		}
 
 		Offset offset = new Offset();
@@ -56,18 +69,20 @@ public class MappedKeyValue extends KeyValue {
 		byte[] value = takeBytes(offset, bytes, lengths);
 		byte[] rangeBegin = takeBytes(offset, bytes, lengths);
 		byte[] rangeEnd = takeBytes(offset, bytes, lengths);
+		byte[] boundaryAndExistBytes = takeBytes(offset, bytes, lengths);
+		int boundaryAndExist = ByteBuffer.wrap(boundaryAndExistBytes).order(ByteOrder.LITTLE_ENDIAN).getInt();
 
-		if ((lengths.length - 4) % 2 != 0) {
+		if ((lengths.length - TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) % 2 != 0) {
 			throw new IllegalArgumentException("There needs to be an even number of lengths!");
 		}
-		int count = (lengths.length - 4) / 2;
+		int count = (lengths.length - TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) / 2;
 		List<KeyValue> rangeResult = new ArrayList<>(count);
 		for (int i = 0; i < count; i++) {
 			byte[] k = takeBytes(offset, bytes, lengths);
 			byte[] v = takeBytes(offset, bytes, lengths);
 			rangeResult.add(new KeyValue(k, v));
 		}
-		return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult);
+		return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult, boundaryAndExist);
 	}
 
 	static class Offset {
@@ -96,13 +111,15 @@ public class MappedKeyValue extends KeyValue {
 		MappedKeyValue rhs = (MappedKeyValue) obj;
 		return Arrays.equals(rangeBegin, rhs.rangeBegin) 
 				&& Arrays.equals(rangeEnd, rhs.rangeEnd)
-				&& Objects.equals(rangeResult, rhs.rangeResult);
+				&& Objects.equals(rangeResult, rhs.rangeResult)
+				&& boundaryAndExist == rhs.boundaryAndExist;
 	}
 
 	@Override
 	public int hashCode() {
 		int hashForResult = rangeResult == null ? 0 : rangeResult.hashCode();
-		return 17 + (29 * hashForResult + 37 * Arrays.hashCode(rangeBegin) + Arrays.hashCode(rangeEnd));
+		return 17 +
+		    (29 * hashForResult + boundaryAndExist + 37 * Arrays.hashCode(rangeBegin) + Arrays.hashCode(rangeEnd));
 	}
 
 	@Override
@@ -111,6 +128,7 @@ public class MappedKeyValue extends KeyValue {
 		sb.append("rangeBegin=").append(ByteArrayUtil.printable(rangeBegin));
 		sb.append(", rangeEnd=").append(ByteArrayUtil.printable(rangeEnd));
 		sb.append(", rangeResult=").append(rangeResult);
+		sb.append(", boundaryAndExist=").append(boundaryAndExist);
 		sb.append('}');
 		return super.toString() + "->" + sb.toString();
 	}

bindings/java/src/main/com/apple/foundationdb/MappedRangeQuery.java

@@ -53,18 +53,21 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 	private final byte[] mapper; // Nonnull
 	private final boolean snapshot;
 	private final int rowLimit;
+	private final int matchIndex;
 	private final boolean reverse;
 	private final StreamingMode streamingMode;
 	private final EventKeeper eventKeeper;
 
 	MappedRangeQuery(FDBTransaction transaction, boolean isSnapshot, KeySelector begin, KeySelector end, byte[] mapper,
-	                 int rowLimit, boolean reverse, StreamingMode streamingMode, EventKeeper eventKeeper) {
+	                 int rowLimit, int matchIndex, boolean reverse, StreamingMode streamingMode,
+	                 EventKeeper eventKeeper) {
 		this.tr = transaction;
 		this.begin = begin;
 		this.end = end;
 		this.mapper = mapper;
 		this.snapshot = isSnapshot;
 		this.rowLimit = rowLimit;
+		this.matchIndex = matchIndex;
 		this.reverse = reverse;
 		this.streamingMode = streamingMode;
 		this.eventKeeper = eventKeeper;
@@ -88,14 +91,14 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 
 			FutureMappedResults range =
 			    tr.getMappedRange_internal(this.begin, this.end, this.mapper, this.rowLimit, 0,
-			                               StreamingMode.EXACT.code(), 1, this.snapshot, this.reverse);
+			                               StreamingMode.EXACT.code(), 1, this.snapshot, this.reverse, this.matchIndex);
 			return range.thenApply(result -> result.get().values).whenComplete((result, e) -> range.close());
 		}
 
 		// If the streaming mode is not EXACT, simply collect the results of an
 		// iteration into a list
 		return AsyncUtil.collect(
-		    new MappedRangeQuery(tr, snapshot, begin, end, mapper, rowLimit, reverse, mode, eventKeeper),
+		    new MappedRangeQuery(tr, snapshot, begin, end, mapper, rowLimit, matchIndex, reverse, mode, eventKeeper),
 		    tr.getExecutor());
 	}
 
@@ -106,7 +109,7 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 	 */
 	@Override
 	public AsyncRangeIterator iterator() {
-		return new AsyncRangeIterator(this.rowLimit, this.reverse, this.streamingMode);
+		return new AsyncRangeIterator(this.rowLimit, this.matchIndex, this.reverse, this.streamingMode);
 	}
 
 	private class AsyncRangeIterator implements AsyncIterator<MappedKeyValue> {
@@ -114,6 +117,7 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 		private final boolean rowsLimited;
 		private final boolean reverse;
 		private final StreamingMode streamingMode;
+		private final int matchIndex;
 
 		// There is the chance for parallelism in the two "chunks" for fetched data
 		private MappedRangeResult chunk = null;
@@ -131,12 +135,13 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 		private CompletableFuture<Boolean> nextFuture;
 		private boolean isCancelled = false;
 
-		private AsyncRangeIterator(int rowLimit, boolean reverse, StreamingMode streamingMode) {
+		private AsyncRangeIterator(int rowLimit, int matchIndex, boolean reverse, StreamingMode streamingMode) {
 			this.begin = MappedRangeQuery.this.begin;
 			this.end = MappedRangeQuery.this.end;
 			this.rowsLimited = rowLimit != 0;
 			this.rowsRemaining = rowLimit;
 			this.reverse = reverse;
+			this.matchIndex = matchIndex;
 			this.streamingMode = streamingMode;
 
 			startNextFetch();
@@ -217,8 +222,9 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
 
 			nextFuture = new CompletableFuture<>();
 			final long sTime = System.nanoTime();
-			fetchingChunk = tr.getMappedRange_internal(begin, end, mapper, rowsLimited ? rowsRemaining : 0, 0,
+			fetchingChunk =
-			                                           streamingMode.code(), ++iteration, snapshot, reverse);
+			    tr.getMappedRange_internal(begin, end, mapper, rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(),
+			                               ++iteration, snapshot, reverse, matchIndex);
 
 			BiConsumer<MappedRangeResultInfo, Throwable> cons = new FetchComplete(fetchingChunk, nextFuture);
 			if (eventKeeper != null) {

bindings/java/src/main/com/apple/foundationdb/MappedRangeResultDirectBufferIterator.java

@@ -51,6 +51,8 @@ class MappedRangeResultDirectBufferIterator extends DirectBufferIterator impleme
 		final byte[] value = getString();
 		final byte[] rangeBegin = getString();
 		final byte[] rangeEnd = getString();
+		final byte[] boundaryAndExistBytes = getString();
+		final int boundaryAndExist = ByteBuffer.wrap(boundaryAndExistBytes).getInt();
 		final int rangeResultSize = byteBuffer.getInt();
 		List<KeyValue> rangeResult = new ArrayList();
 		for (int i = 0; i < rangeResultSize; i++) {
@@ -59,7 +61,7 @@ class MappedRangeResultDirectBufferIterator extends DirectBufferIterator impleme
 			rangeResult.add(new KeyValue(k, v));
 		}
 		current += 1;
-		return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult);
+		return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult, boundaryAndExist);
 	}
 
 	private byte[] getString() {

bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java

@@ -460,7 +460,7 @@ public interface ReadTransaction extends ReadTransactionContext {
 	 * @return a handle to access the results of the asynchronous call
 	 */
 	AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper, int limit,
-	                                                 boolean reverse, StreamingMode mode);
+	                                             int matchIndex, boolean reverse, StreamingMode mode);
 
 	/**
 	 * Gets an estimate for the number of bytes stored in the given range.

bindings/python/tests/fdbcli_tests.py

@@ -215,6 +215,26 @@ def kill(logger):
     assert new_generation > old_generation
 
 
+@enable_logging()
+def killall(logger):
+    # test is designed to make sure 'kill all' sends all requests simultaneously
+    old_generation = get_value_from_status_json(False, 'cluster', 'generation')
+    # This is currently an issue with fdbcli,
+    # where you need to first run 'kill' to initialize processes' list
+    # and then specify the certain process to kill
+    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env)
+    output, error = process.communicate(input='kill; kill all; sleep 1\n'.encode())
+    logger.debug(output)
+    # wait for a second for the cluster recovery
+    time.sleep(1)
+    new_generation = get_value_from_status_json(True, 'cluster', 'generation')
+    logger.debug("Old generation: {}, New generation: {}".format(old_generation, new_generation))
+    # Make sure the kill is not happening sequentially
+    # Pre: each recovery will increase the generated number by 2
+    # Relax the condition to allow one additional recovery happening when we fetched the old generation
+    assert new_generation <= (old_generation + 4)
+
+
 @enable_logging()
 def suspend(logger):
     if not shutil.which("pidof"):
@@ -582,6 +602,7 @@ def triggerddteaminfolog(logger):
     output = run_fdbcli_command('triggerddteaminfolog')
     assert output == 'Triggered team info logging in data distribution.'
 
+
 @enable_logging()
 def tenants(logger):
     output = run_fdbcli_command('listtenants')
@@ -610,7 +631,7 @@ def tenants(logger):
     assert len(lines) == 2
     assert lines[0].strip().startswith('id: ')
     assert lines[1].strip().startswith('prefix: ')
-    
+
     output = run_fdbcli_command('usetenant')
     assert output == 'Using the default tenant'
 
@@ -652,7 +673,8 @@ def tenants(logger):
     assert lines[3] == '`tenant_test\' is `default_tenant\''
 
     process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
-    cmd_sequence = ['writemode on', 'usetenant tenant', 'clear tenant_test', 'deletetenant tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
+    cmd_sequence = ['writemode on', 'usetenant tenant', 'clear tenant_test',
+                    'deletetenant tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
     output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
 
     lines = output.decode().strip().split('\n')[-7:]
@@ -680,6 +702,7 @@ def tenants(logger):
 
     run_fdbcli_command('writemode on; clear tenant_test')
 
+
 if __name__ == '__main__':
     parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
                             description="""
@@ -731,5 +754,6 @@ if __name__ == '__main__':
         assert args.process_number > 1, "Process number should be positive"
         coordinators()
         exclude()
+        killall()
         # TODO: fix the failure where one process is not available after setclass call
-        #setclass()
+        # setclass()

cmake/CompileBoost.cmake

@@ -123,7 +123,7 @@ set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores an
 
 if(Boost_FOUND AND Boost_filesystem_FOUND AND Boost_context_FOUND AND NOT FORCE_BOOST_BUILD)
   add_library(boost_target INTERFACE)
-  target_link_libraries(boost_target INTERFACE Boost::boost Boost::context_FOUND Boost::filesystem)
+  target_link_libraries(boost_target INTERFACE Boost::boost Boost::context Boost::filesystem)
 elseif(WIN32)
   message(FATAL_ERROR "Could not find Boost")
 else()

cmake/FDBComponents.cmake

@@ -217,7 +217,7 @@ set(DEFAULT_COROUTINE_IMPL boost)
 if(WIN32)
   # boost coroutine not available in windows build environment for now.
   set(DEFAULT_COROUTINE_IMPL libcoro)
-elseif(NOT APPLE AND NOT USE_SANITIZER AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
+elseif(NOT APPLE AND NOT USE_ASAN AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
   # revert to libcoro for x86 linux while we investigate a performance regression
   set(DEFAULT_COROUTINE_IMPL libcoro)
 endif()

contrib/CMakeLists.txt

@@ -5,3 +5,4 @@ if(NOT WIN32)
   add_subdirectory(TraceLogHelper)
   add_subdirectory(TestHarness)
 endif()
+add_subdirectory(mockkms)

contrib/TestHarness/Program.cs

@@ -376,11 +376,13 @@ namespace SummarizeTest
                     bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
                     bool useToml = File.Exists(testFile + "-1.toml");
                     string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt";
-                    result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
+                    bool useValgrindRunOne = useValgrind && firstServerName == fdbserverName;
+                    bool useValgrindRunTwo = useValgrind && secondServerName == fdbserverName;
+                    result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrindRunOne, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
                     if (result == 0)
                     {
                         string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt";
-                        result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
+                        result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrindRunTwo, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
                     }
                 }
                 else
@@ -458,7 +460,7 @@ namespace SummarizeTest
                             role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg);
                     }
                     if (restarting) args = args + " --restarting";
-                    if (useValgrind && !willRestart)
+                    if (useValgrind)
                     {
                         valgrindOutputFile = string.Format("valgrind-{0}.xml", seed);
                         process.StartInfo.FileName = "valgrind";

contrib/mockkms/CMakeLists.txt

@@ -0,0 +1,18 @@
+if(WITH_GO_BINDING)
+  set(MOCK_KMS_SRC fault_injection.go get_encryption_keys.go mock_kms.go utils.go)
+  set(MOCK_KMS_TEST_SRC ${MOCK_KMS_SRC} mockkms_test.go)
+  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms
+    COMMAND go build -o ${CMAKE_BINARY_DIR}/bin/mockkms ${MOCK_KMS_SRC}
+    DEPENDS ${MOCK_KMS_SRC}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_custom_target(mockkms ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms)
+  fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/mockkms DESTINATION bin COMPONENT server)
+
+  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms_test
+    COMMAND go test -c -o ${CMAKE_BINARY_DIR}/bin/mockkms_test ${MOCK_KMS_TEST_SRC}
+    DEPENDS ${MOCK_KMS_TEST_SRC}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_custom_target(mockkms_test ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms_test)
+  add_test(NAME mockkms COMMAND ${CMAKE_BINARY_DIR}/bin/mockkms_test)
+
+endif()

contrib/mockkms/fault_injection.go

@@ -0,0 +1,179 @@
+/*
+ * fault_injection.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Interface supports client to inject fault(s)
+// Module enables a client to update { FaultLocation -> FaultStatus } mapping in a
+// thread-safe manner, however, client is responsible to synchronize fault status
+// updates across 'getEncryptionKeys' REST requests to obtain predictable results.
+
+package main
+
+import (
+    "encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"sync"
+)
+
+type Fault struct {
+	Location int		 `json:"fault_location"`
+	Enable bool 		 `json:"enable_fault"`
+}
+
+type FaultInjectionRequest struct {
+	Faults []Fault   	 `json:"faults"`
+}
+
+type FaultInjectionResponse struct {
+	Faults []Fault   	 `json:"faults"`
+}
+
+type faultLocMap struct {
+	locMap map[int]bool
+	rwLock sync.RWMutex
+}
+
+var (
+	faultLocMapInstance *faultLocMap		// Singleton mapping of { FaultLocation -> FaultStatus }
+)
+
+// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
+func NewFaultLocMap() *faultLocMap {
+	if faultLocMapInstance == nil {
+		faultLocMapInstance = &faultLocMap{}
+
+		faultLocMapInstance.rwLock = sync.RWMutex{}
+		faultLocMapInstance.locMap = map[int]bool {
+			READ_HTTP_REQUEST_BODY      : false,
+			UNMARSHAL_REQUEST_BODY_JSON : false,
+			UNSUPPORTED_QUERY_MODE      : false,
+			PARSE_HTTP_REQUEST          : false,
+			MARSHAL_RESPONSE            : false,
+		}
+	}
+	return faultLocMapInstance
+}
+
+func getLocFaultStatus(loc int) (val bool, found bool) {
+	if faultLocMapInstance == nil {
+		panic("FaultLocMap not intialized")
+		os.Exit(1)
+	}
+
+	faultLocMapInstance.rwLock.RLock()
+	defer faultLocMapInstance.rwLock.RUnlock()
+	val, found = faultLocMapInstance.locMap[loc]
+	if !found {
+		return
+	}
+	return
+}
+
+func updateLocFaultStatuses(faults []Fault) (updated []Fault, err error) {
+	if faultLocMapInstance == nil {
+		panic("FaultLocMap not intialized")
+		os.Exit(1)
+	}
+
+	updated = []Fault{}
+	err = nil
+
+	faultLocMapInstance.rwLock.Lock()
+	defer faultLocMapInstance.rwLock.Unlock()
+	for i := 0; i < len(faults); i++ {
+		fault := faults[i]
+
+		oldVal, found := faultLocMapInstance.locMap[fault.Location]
+		if !found {
+			err = fmt.Errorf("Unknown fault_location '%d'", fault.Location)
+			return
+		}
+		faultLocMapInstance.locMap[fault.Location] = fault.Enable
+		log.Printf("Update Location '%d' oldVal '%t' newVal '%t'", fault.Location, oldVal, fault.Enable)
+	}
+
+	// return the updated faultLocMap
+	for loc, enable := range faultLocMapInstance.locMap {
+		var f Fault
+		f.Location = loc
+		f.Enable = enable
+		updated = append(updated, f)
+	}
+	return
+}
+
+func jsonifyFaultArr(w http.ResponseWriter, faults []Fault) (jResp string) {
+	resp := FaultInjectionResponse{
+		Faults:	faults,
+	}
+
+	mResp, err := json.Marshal(resp)
+	if err != nil {
+		log.Printf("Error marshaling response '%s'", err.Error())
+		sendErrorResponse(w, err)
+		return
+	}
+	jResp = string(mResp)
+	return
+}
+
+func updateFaultLocMap(w http.ResponseWriter, faults []Fault) {
+	updated , err := updateLocFaultStatuses(faults)
+	if err != nil {
+		sendErrorResponse(w, err)
+		return
+	}
+
+	fmt.Fprintf(w, jsonifyFaultArr(w, updated))
+}
+
+func shouldInjectFault(loc int) bool {
+	status, found := getLocFaultStatus(loc)
+	if !found {
+		log.Printf("Unknown fault_location '%d'", loc)
+		return false
+	}
+	return status
+}
+
+func handleUpdateFaultInjection(w http.ResponseWriter, r *http.Request) {
+	byteArr, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("Http request body read error '%s'", err.Error())
+		sendErrorResponse(w, err)
+		return
+	}
+
+	req := FaultInjectionRequest{}
+	err = json.Unmarshal(byteArr, &req)
+	if err != nil {
+		log.Printf("Error parsing FaultInjectionRequest '%s'", string(byteArr))
+		sendErrorResponse(w, err)
+	}
+	updateFaultLocMap(w, req.Faults)
+}
+
+func initFaultLocMap() {
+	faultLocMapInstance = NewFaultLocMap()
+    log.Printf("FaultLocMap int done")
+}

contrib/mockkms/get_encryption_keys.go

@@ -0,0 +1,321 @@
+/*
+ * get_encryption_keys.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// GetEncryptionKeys handler
+// Handler is resposible for the following:
+// 1. Parse the incoming HttpRequest and validate JSON request structural sanity
+// 2. Ability to handle getEncryptionKeys by 'KeyId' or 'DomainId' as requested
+// 3. Ability to inject faults if requested
+
+package main
+
+import (
+    "encoding/json"
+    "fmt"
+    "io/ioutil"
+    "log"
+    "math/rand"
+    "net/http"
+)
+
+type CipherDetailRes struct {
+    BaseCipherId uint64                    `json:"base_cipher_id"`
+    EncryptDomainId int64                  `json:"encrypt_domain_id"`
+    BaseCipher string                      `json:"base_cipher"`
+}
+
+type ValidationToken struct {
+    TokenName string                       `json:"token_name"`
+    TokenValue string                      `json:"token_value"`
+}
+
+type CipherDetailReq struct {
+    BaseCipherId uint64                   `json:"base_cipher_id"`
+    EncryptDomainId int64                 `json:"encrypt_domain_id"`
+}
+
+type GetEncryptKeysResponse struct {
+    CipherDetails []CipherDetailRes        `json:"cipher_key_details"`
+    KmsUrls []string                       `json:"kms_urls"`
+}
+
+type GetEncryptKeysRequest struct {
+    QueryMode string                      `json:"query_mode"`
+    CipherDetails []CipherDetailReq       `json:"cipher_key_details"`
+    ValidationTokens []ValidationToken    `json:"validation_tokens"`
+    RefreshKmsUrls bool                   `json:"refresh_kms_urls"`
+}
+
+type cipherMapInstanceSingleton map[uint64][]byte
+
+const (
+	READ_HTTP_REQUEST_BODY = iota
+	UNMARSHAL_REQUEST_BODY_JSON
+	UNSUPPORTED_QUERY_MODE
+	PARSE_HTTP_REQUEST
+	MARSHAL_RESPONSE
+)
+
+const (
+    maxCipherKeys = uint64(1024*1024)                   // Max cipher keys
+    maxCipherSize = 16                                  // Max cipher buffer size
+)
+
+var (
+    cipherMapInstance cipherMapInstanceSingleton        // Singleton mapping of { baseCipherId -> baseCipher }
+)
+
+// const mapping of { Location -> errorString }
+func errStrMap() func(int) string {
+    _errStrMap := map[int]string {
+        READ_HTTP_REQUEST_BODY : "Http request body read error",
+        UNMARSHAL_REQUEST_BODY_JSON : "Http request body unmarshal error",
+        UNSUPPORTED_QUERY_MODE : "Unsupported query_mode",
+        PARSE_HTTP_REQUEST : "Error parsing GetEncryptionKeys request",
+        MARSHAL_RESPONSE : "Error marshaling response",
+    }
+
+    return func(key int) string {
+        return _errStrMap[key]
+    }
+}
+
+// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
+func NewCipherMap(maxKeys uint64, cipherSize int) cipherMapInstanceSingleton {
+    if cipherMapInstance == nil {
+        cipherMapInstance = make(map[uint64][]byte)
+
+        for i := uint64(1); i<= maxKeys; i++ {
+            cipher := make([]byte, cipherSize)
+            rand.Read(cipher)
+            cipherMapInstance[i] = cipher
+        }
+        log.Printf("KMS cipher map populate done, maxCiphers '%d'", maxCipherKeys)
+    }
+    return cipherMapInstance
+}
+
+func getKmsUrls() (urls []string) {
+    urlCount := rand.Intn(5) + 1
+    for i := 1; i <= urlCount; i++ {
+        url := fmt.Sprintf("https://KMS/%d:%d:%d:%d", i, i, i, i)
+        urls = append(urls, url)
+    }
+    return
+}
+
+func isEncryptDomainIdValid(id int64) bool {
+    if id > 0 || id == -1 || id == -2 {
+        return true
+    }
+    return false
+}
+
+func abs(x int64) int64 {
+    if x < 0 {
+        return -x
+    }
+    return x
+}
+
+func getBaseCipherIdFromDomainId(domainId int64) (baseCipherId uint64) {
+    baseCipherId = uint64(1) + uint64(abs(domainId)) % maxCipherKeys
+    return
+}
+
+func getEncryptionKeysByKeyIds(w http.ResponseWriter, byteArr []byte) {
+    req := GetEncryptKeysRequest{}
+    err := json.Unmarshal(byteArr, &req)
+    if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
+        var e error
+        if shouldInjectFault(PARSE_HTTP_REQUEST) {
+            e = fmt.Errorf("[FAULT] %s %s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
+        } else {
+            e = fmt.Errorf("%s %s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    var details []CipherDetailRes
+    for i := 0; i < len(req.CipherDetails); i++ {
+        var baseCipherId = uint64(req.CipherDetails[i].BaseCipherId)
+
+        var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
+        if !isEncryptDomainIdValid(encryptDomainId) {
+            e := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
+            sendErrorResponse(w, e)
+            return
+        }
+
+        cipher, found := cipherMapInstance[baseCipherId]
+        if !found {
+            e := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
+            sendErrorResponse(w, e)
+            return
+        }
+
+        var detail = CipherDetailRes {
+            BaseCipherId:       baseCipherId,
+            EncryptDomainId:    encryptDomainId,
+            BaseCipher:         string(cipher),
+        }
+        details = append(details, detail)
+    }
+
+    var urls []string
+    if req.RefreshKmsUrls {
+        urls = getKmsUrls()
+    }
+
+    resp := GetEncryptKeysResponse{
+        CipherDetails:  details,
+        KmsUrls:        urls,
+    }
+
+    mResp, err := json.Marshal(resp)
+    if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
+        var e error
+        if shouldInjectFault(MARSHAL_RESPONSE) {
+            e = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
+        } else {
+            e = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    fmt.Fprintf(w, string(mResp))
+}
+
+func getEncryptionKeysByDomainIds(w http.ResponseWriter, byteArr []byte) {
+	req := GetEncryptKeysRequest{}
+    err := json.Unmarshal(byteArr, &req)
+    if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
+        var e error
+        if shouldInjectFault(PARSE_HTTP_REQUEST) {
+            e = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
+        } else {
+            e = fmt.Errorf("%s '%s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    var details []CipherDetailRes
+    for i := 0; i < len(req.CipherDetails); i++ {
+        var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
+        if !isEncryptDomainIdValid(encryptDomainId) {
+            e := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
+            sendErrorResponse(w, e)
+            return
+        }
+
+        var baseCipherId = getBaseCipherIdFromDomainId(encryptDomainId)
+        cipher, found := cipherMapInstance[baseCipherId]
+        if !found {
+            e := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
+            sendErrorResponse(w, e)
+            return
+        }
+
+        var detail = CipherDetailRes {
+            BaseCipherId:       baseCipherId,
+            EncryptDomainId:    encryptDomainId,
+            BaseCipher:         string(cipher),
+        }
+        details = append(details, detail)
+    }
+
+    var urls []string
+    if req.RefreshKmsUrls {
+        urls = getKmsUrls()
+    }
+
+    resp := GetEncryptKeysResponse{
+        CipherDetails:  details,
+        KmsUrls:        urls,
+    }
+
+    mResp, err := json.Marshal(resp)
+    if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
+        var e error
+        if shouldInjectFault(MARSHAL_RESPONSE) {
+            e = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
+        } else {
+            e = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    fmt.Fprintf(w, string(mResp))
+}
+
+func handleGetEncryptionKeys(w http.ResponseWriter, r *http.Request) {
+    byteArr, err := ioutil.ReadAll(r.Body)
+    if err != nil || shouldInjectFault(READ_HTTP_REQUEST_BODY) {
+        var e error
+        if shouldInjectFault(READ_HTTP_REQUEST_BODY) {
+            e = fmt.Errorf("[FAULT] %s", errStrMap()(READ_HTTP_REQUEST_BODY))
+        } else {
+            e = fmt.Errorf("%s err '%v'", errStrMap()(READ_HTTP_REQUEST_BODY), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    var arbitrary_json map[string]interface{}
+    err = json.Unmarshal(byteArr, &arbitrary_json)
+    if err != nil || shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
+        var e error
+        if shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
+            e = fmt.Errorf("[FAULT] %s", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON))
+        } else {
+            e = fmt.Errorf("%s err '%v'", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON), err)
+        }
+        log.Println(e.Error())
+        sendErrorResponse(w, e)
+        return
+    }
+
+    if shouldInjectFault(UNSUPPORTED_QUERY_MODE) {
+        err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
+        sendErrorResponse(w, err)
+        return
+    } else if arbitrary_json["query_mode"] == "lookupByKeyId" {
+        getEncryptionKeysByKeyIds(w, byteArr)
+    } else if arbitrary_json["query_mode"] == "lookupByDomainId" {
+        getEncryptionKeysByDomainIds(w, byteArr)
+    } else {
+        err = fmt.Errorf("%s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
+        sendErrorResponse(w, err)
+        return
+    }
+}
+
+func initEncryptCipherMap() {
+    cipherMapInstance = NewCipherMap(maxCipherKeys, maxCipherSize)
+}

contrib/mockkms/mock_kms.go

@@ -0,0 +1,66 @@
+/*
+ * mock_kms.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// FoundationDB Mock KMS (Key Management Solution/Service) interface
+// Interface runs an HTTP server handling REST calls simulating FDB communications
+// with an external KMS.
+
+package main
+
+import (
+    "log"
+    "math/rand"
+    "net/http"
+    "sync"
+    "time"
+)
+
+// KMS supported endpoints
+const (
+    getEncryptionKeysEndpoint    = "/getEncryptionKeys"
+    updateFaultInjectionEndpoint = "/updateFaultInjection"
+)
+
+// Routine is responsible to instantiate data-structures necessary for MockKMS functioning
+func init () {
+    var wg sync.WaitGroup
+
+    wg.Add(2)
+    go func(){
+        initEncryptCipherMap()
+        wg.Done()
+    }()
+    go func(){
+        initFaultLocMap()
+        wg.Done()
+    }()
+
+    wg.Wait()
+
+	rand.Seed(time.Now().UTC().UnixNano())
+}
+
+func main() {
+    http.NewServeMux()
+    http.HandleFunc(getEncryptionKeysEndpoint, handleGetEncryptionKeys)
+    http.HandleFunc(updateFaultInjectionEndpoint, handleUpdateFaultInjection)
+
+    log.Fatal(http.ListenAndServe(":5001", nil))
+}

contrib/mockkms/mockkms_test.go

@@ -0,0 +1,302 @@
+/*
+ * mockkms_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// MockKMS unit tests, the coverage includes:
+// 1. Mock HttpRequest creation and HttpResponse writer.
+// 2. Construct fake request to validate the following scenarions:
+//  2.1. Request with "unsupported query mode"
+//  2.2. Get encryption keys by KeyIds; with and without 'RefreshKmsUrls' flag.
+//  2.2. Get encryption keys by DomainIds; with and without 'RefreshKmsUrls' flag.
+//  2.3. Random fault injection and response validation
+
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+const (
+	ByKeyIdReqWithRefreshUrls = `{
+		"query_mode": "lookupByKeyId",
+		"cipher_key_details": [
+			{
+				"base_cipher_id": 77,
+				"encrypt_domain_id": 76
+			},
+			{
+				"base_cipher_id": 2,
+				"encrypt_domain_id": -1
+			}
+		],
+		"validation_tokens": [
+			{
+				"token_name": "1",
+				"token_value":"12344"
+			},
+			{
+				"token_name": "2",
+				"token_value":"12334"
+			}
+		],
+		"refresh_kms_urls": true
+	}`
+	ByKeyIdReqWithoutRefreshUrls = `{
+		"query_mode": "lookupByKeyId",
+		"cipher_key_details": [
+			{
+				"base_cipher_id": 77,
+				"encrypt_domain_id": 76
+			},
+			{
+				"base_cipher_id": 2,
+				"encrypt_domain_id": -1
+			}
+		],
+		"validation_tokens": [
+			{
+				"token_name": "1",
+				"token_value":"12344"
+			},
+			{
+				"token_name": "2",
+				"token_value":"12334"
+			}
+		],
+		"refresh_kms_urls": false
+	}`
+	ByDomainIdReqWithRefreshUrls = `{
+		"query_mode": "lookupByDomainId",
+		"cipher_key_details": [
+			{
+				"encrypt_domain_id": 76
+			},
+			{
+				"encrypt_domain_id": -1
+			}
+		],
+		"validation_tokens": [
+			{
+				"token_name": "1",
+				"token_value":"12344"
+			},
+			{
+				"token_name": "2",
+				"token_value":"12334"
+			}
+		],
+		"refresh_kms_urls": true
+	}`
+	ByDomainIdReqWithoutRefreshUrls = `{
+		"query_mode": "lookupByDomainId",
+		"cipher_key_details": [
+			{
+				"encrypt_domain_id": 76
+			},
+			{
+				"encrypt_domain_id": -1
+			}
+		],
+		"validation_tokens": [
+			{
+				"token_name": "1",
+				"token_value":"12344"
+			},
+			{
+				"token_name": "2",
+				"token_value":"12334"
+			}
+		],
+		"refresh_kms_urls": false
+	}`
+	UnsupportedQueryMode= `{
+		"query_mode": "foo_mode",
+		"cipher_key_details": [
+			{
+				"encrypt_domain_id": 76
+			},
+			{
+				"encrypt_domain_id": -1
+			}
+		],
+		"validation_tokens": [
+			{
+				"token_name": "1",
+				"token_value":"12344"
+			},
+			{
+				"token_name": "2",
+				"token_value":"12334"
+			}
+		],
+		"refresh_kms_urls": false
+	}`
+)
+
+func unmarshalValidResponse(data []byte, t *testing.T) (resp GetEncryptKeysResponse) {
+	resp = GetEncryptKeysResponse{}
+	err := json.Unmarshal(data, &resp)
+	if err != nil {
+		t.Errorf("Error unmarshaling valid response '%s' error '%v'", string(data), err)
+		t.Fail()
+	}
+	return
+}
+
+func unmarshalErrorResponse(data []byte, t *testing.T) (resp ErrorResponse) {
+	resp = ErrorResponse{}
+	err := json.Unmarshal(data, &resp)
+	if err != nil {
+		t.Errorf("Error unmarshaling error response resp '%s' error '%v'", string(data), err)
+		t.Fail()
+	}
+	return
+}
+
+func checkGetEncyptKeysResponseValidity(resp GetEncryptKeysResponse, t *testing.T) {
+	if len(resp.CipherDetails) != 2 {
+		t.Errorf("Unexpected CipherDetails count, expected '%d' actual '%d'", 2, len(resp.CipherDetails))
+		t.Fail()
+	}
+
+	baseCipherIds := [...]uint64 {uint64(77), uint64(2)}
+	encryptDomainIds := [...]int64 {int64(76), int64(-1)}
+
+	for i := 0; i < len(resp.CipherDetails); i++ {
+		if resp.CipherDetails[i].BaseCipherId != baseCipherIds[i] {
+			t.Errorf("Mismatch BaseCipherId, expected '%d' actual '%d'", baseCipherIds[i], resp.CipherDetails[i].BaseCipherId)
+			t.Fail()
+		}
+		if resp.CipherDetails[i].EncryptDomainId != encryptDomainIds[i] {
+			t.Errorf("Mismatch EncryptDomainId, expected '%d' actual '%d'", encryptDomainIds[i], resp.CipherDetails[i].EncryptDomainId)
+			t.Fail()
+		}
+		if len(resp.CipherDetails[i].BaseCipher) == 0 {
+			t.Error("Empty BaseCipher")
+			t.Fail()
+		}
+	}
+}
+
+func runQueryExpectingErrorResponse(payload string, url string, errSubStr string, t *testing.T) {
+	body := strings.NewReader(payload)
+	req := httptest.NewRequest(http.MethodPost, url, body)
+	w := httptest.NewRecorder()
+	handleGetEncryptionKeys(w, req)
+	res := w.Result()
+	defer res.Body.Close()
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Errorf("Error %v", err)
+	}
+
+	resp := unmarshalErrorResponse(data, t)
+	if !strings.Contains(resp.Err.Detail, errSubStr) {
+		t.Errorf("Unexpected error response '%s'", resp.Err.Detail)
+		t.Fail()
+	}
+}
+
+func runQueryExpectingValidResponse(payload string, url string, t *testing.T) {
+	body := strings.NewReader(payload)
+	req := httptest.NewRequest(http.MethodPost, url, body)
+	w := httptest.NewRecorder()
+	handleGetEncryptionKeys(w, req)
+	res := w.Result()
+	defer res.Body.Close()
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Errorf("Error %v", err)
+	}
+
+	resp := unmarshalValidResponse(data, t)
+	checkGetEncyptKeysResponseValidity(resp, t)
+}
+
+func TestUnsupportedQueryMode(t *testing.T) {
+	runQueryExpectingErrorResponse(UnsupportedQueryMode, getEncryptionKeysEndpoint, errStrMap()(UNSUPPORTED_QUERY_MODE), t)
+}
+
+func TestGetEncryptionKeysByKeyIdsWithRefreshUrls(t *testing.T) {
+	runQueryExpectingValidResponse(ByKeyIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
+}
+
+func TestGetEncryptionKeysByKeyIdsWithoutRefreshUrls(t *testing.T) {
+	runQueryExpectingValidResponse(ByKeyIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
+}
+
+func TestGetEncryptionKeysByDomainIdsWithRefreshUrls(t *testing.T) {
+	runQueryExpectingValidResponse(ByDomainIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
+}
+
+func TestGetEncryptionKeysByDomainIdsWithoutRefreshUrls(t *testing.T) {
+	runQueryExpectingValidResponse(ByDomainIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
+}
+
+func TestFaultInjection(t *testing.T) {
+	numIterations := rand.Intn(701) + 86
+
+	for i := 0; i < numIterations; i++ {
+		loc := rand.Intn(MARSHAL_RESPONSE + 1)
+		f := Fault{}
+		f.Location = loc
+		f.Enable = true
+
+		var faults []Fault
+		faults = append(faults, f)
+		fW := httptest.NewRecorder()
+		body := strings.NewReader(jsonifyFaultArr(fW, faults))
+		fReq := httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
+		handleUpdateFaultInjection(fW, fReq)
+		if !shouldInjectFault(loc) {
+			t.Errorf("Expected fault enabled for loc '%d'", loc)
+			t.Fail()
+		}
+
+		var payload string
+		lottery := rand.Intn(100)
+		if lottery < 25 {
+			payload = ByKeyIdReqWithRefreshUrls
+		} else if lottery >= 25 && lottery < 50 {
+			payload = ByKeyIdReqWithoutRefreshUrls
+		} else if lottery >= 50 && lottery < 75 {
+			payload = ByDomainIdReqWithRefreshUrls
+		} else {
+			payload = ByDomainIdReqWithoutRefreshUrls
+		}
+		runQueryExpectingErrorResponse(payload, getEncryptionKeysEndpoint, errStrMap()(loc), t)
+
+		// reset Fault
+		faults[0].Enable = false
+		fW = httptest.NewRecorder()
+		body = strings.NewReader(jsonifyFaultArr(fW, faults))
+		fReq = httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
+		handleUpdateFaultInjection(fW, fReq)
+		if shouldInjectFault(loc) {
+			t.Errorf("Expected fault disabled for loc '%d'", loc)
+			t.Fail()
+		}
+	}
+}

contrib/mockkms/utils.go

@@ -0,0 +1,51 @@
+/*
+ * utils.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+    "encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+)
+
+type ErrorDetail struct {
+    Detail string                          `json:"details"`
+}
+
+type ErrorResponse struct {
+    Err ErrorDetail                        `json:"error"`
+}
+
+func sendErrorResponse(w http.ResponseWriter, err error) {
+    e := ErrorDetail{}
+    e.Detail = fmt.Sprintf("Error: %s", err.Error())
+    resp := ErrorResponse{
+        Err:    e,
+    }
+
+    mResp,err  := json.Marshal(resp)
+    if err != nil {
+        log.Printf("Error marshalling error response %s", err.Error())
+        panic(err)
+    }
+    fmt.Fprintf(w, string(mResp))
+}

contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py

@@ -194,7 +194,8 @@ class BaseInfo(object):
         if protocol_version >= PROTOCOL_VERSION_6_3:
             self.dc_id = bb.get_bytes_with_length()
         if protocol_version >= PROTOCOL_VERSION_7_1:
-            self.tenant = bb.get_bytes_with_length()
+            if bb.get_bytes(1):
+                self.tenant = bb.get_bytes_with_length()
 
 class GetVersionInfo(BaseInfo):
     def __init__(self, bb, protocol_version):

documentation/sphinx/source/mr-status-json-schemas.rst.inc

@@ -877,6 +877,9 @@
                "logical_core_utilization":0.4 // computed as cpu_seconds / elapsed_seconds; value may be capped at 0.5 due to hyper-threading
             }
          }
+      },
+      "tenants":{
+         "num_tenants":0
       }
    },
    "client":{

documentation/sphinx/source/release-notes/release-notes-710.rst

@@ -4,6 +4,52 @@
 Release Notes
 #############
 
+7.1.5
+=====
+* Fixed a fdbcli kill bug that was not killing in parallel. `(PR #7150) <https://github.com/apple/foundationdb/pull/7150>`_
+* Fixed a bug that prevents a peer from sending messages on a previously incompatible connection. `(PR #7124) <https://github.com/apple/foundationdb/pull/7124>`_
+* Added rocksdb throttling counters to trace event. `(PR #7096) <https://github.com/apple/foundationdb/pull/7096>`_
+* Added a backtrace before throwing serialization_failed. `(PR #7155) <https://github.com/apple/foundationdb/pull/7155>`_
+
+7.1.4
+=====
+* Fixed a bug that prevents client from connecting to a cluster. `(PR #7060) <https://github.com/apple/foundationdb/pull/7060>`_
+* Fixed a performance bug that overloads Resolver CPU. `(PR #7068) <https://github.com/apple/foundationdb/pull/7068>`_
+* Optimized storage server performance for "get range and flat map" feature. `(PR #7078) <https://github.com/apple/foundationdb/pull/7078>`_
+* Optimized both Proxy performance and Resolver (when version vector is enabled) performance. `(PR #7076) <https://github.com/apple/foundationdb/pull/7076>`_
+* Fixed a key size limit bug when using tenants. `(PR #6986) <https://github.com/apple/foundationdb/pull/6986>`_
+* Fixed operation_failed thrown incorrectly from transactions. `(PR #6993) <https://github.com/apple/foundationdb/pull/6993>`_
+* Fixed a version vector bug when GRV cache is used. `(PR #7057) <https://github.com/apple/foundationdb/pull/7057>`_
+* Fixed orphaned storage server due to force recovery. `(PR #7028) <https://github.com/apple/foundationdb/pull/7028>`_
+* Fixed a bug that a storage server reads stale cluster ID. `(PR #7026) <https://github.com/apple/foundationdb/pull/7026>`_
+* Fixed a storage server exclusion status bug that affects wiggling. `(PR #6984) <https://github.com/apple/foundationdb/pull/6984>`_
+* Fixed a bug that relocate shard tasks move data to a removed team. `(PR #7023) <https://github.com/apple/foundationdb/pull/7023>`_
+* Fixed recruitment thrashing when there are temporarily multiple cluster controllers. `(PR #7001) <https://github.com/apple/foundationdb/pull/7001>`_
+* Fixed change feed deletion due to multiple sources race. `(PR #6987) <https://github.com/apple/foundationdb/pull/6987>`_
+* Fixed TLog crash if more TLogs are absent than the replication factor. `(PR #6991) <https://github.com/apple/foundationdb/pull/6991>`_
+* Added hostname DNS resolution logic for cluster connection string. `(PR #6998) <https://github.com/apple/foundationdb/pull/6998>`_
+* Fixed a limit bug in indexPrefetch. `(PR #7005) <https://github.com/apple/foundationdb/pull/7005>`_
+
+7.1.3
+=====
+* Added logging measuring commit compute duration. `(PR #6906) <https://github.com/apple/foundationdb/pull/6906>`_
+* RocksDb used aggregated property metrics for pending compaction bytes. `(PR #6867) <https://github.com/apple/foundationdb/pull/6867>`_
+* Fixed a perpetual wiggle bug that would not react to a pause. `(PR #6933) <https://github.com/apple/foundationdb/pull/6933>`_
+* Fixed a crash of data distributor. `(PR #6938) <https://github.com/apple/foundationdb/pull/6938>`_
+* Added new c libs to client package. `(PR #6921) <https://github.com/apple/foundationdb/pull/6921>`_
+* Fixed a bug that prevents a cluster from fully recovered state after taking a snapshot. `(PR #6892) <https://github.com/apple/foundationdb/pull/6892>`_
+
+7.1.2
+=====
+* Fixed failing upgrades due to non-persisted initial cluster version. `(PR #6864) <https://github.com/apple/foundationdb/pull/6864>`_
+* Fixed a client load balancing bug because ClientDBInfo may be unintentionally not set. `(PR #6878) <https://github.com/apple/foundationdb/pull/6878>`_
+* Fixed stuck LogRouter due to races of multiple PeekStream requests. `(PR #6870) <https://github.com/apple/foundationdb/pull/6870>`_
+* Fixed a client-side infinite loop due to provisional GRV Proxy ID not set in GetReadVersionReply. `(PR #6849) <https://github.com/apple/foundationdb/pull/6849>`_
+
+7.1.1
+=====
+* Added new c libs to client package. `(PR #6828) <https://github.com/apple/foundationdb/pull/6828>`_
+
 7.1.0
 =====
 

fdbbackup/backup.actor.cpp

@@ -620,6 +620,7 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
 #endif
 	{ OPT_BASEURL, "-b", SO_REQ_SEP },
 	{ OPT_BASEURL, "--base-url", SO_REQ_SEP },
+	{ OPT_PROXY, "--proxy", SO_REQ_SEP },
 	{ OPT_TRACE, "--log", SO_NONE },
 	{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
 	{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
@@ -3336,6 +3337,10 @@ int main(int argc, char* argv[]) {
 		}
 
 		Optional<std::string> proxy;
+		std::string p;
+		if (platform::getEnvironmentVar("HTTP_PROXY", p) || platform::getEnvironmentVar("HTTPS_PROXY", p)) {
+			proxy = p;
+		}
 		std::string destinationContainer;
 		bool describeDeep = false;
 		bool describeTimestamps = false;

fdbcli/ExpensiveDataCheckCommand.actor.cpp

@@ -18,6 +18,8 @@
  * limitations under the License.
  */
 
+#include "boost/algorithm/string.hpp"
+
 #include "fdbcli/fdbcli.actor.h"
 
 #include "fdbclient/FDBOptions.g.h"
@@ -40,8 +42,10 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
     std::vector<StringRef> tokens,
     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
 	state bool result = true;
+	state std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
+		address_interface->clear();
 		wait(getWorkerInterfaces(tr, address_interface));
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
@@ -57,20 +61,26 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 		}
 		printf("\n");
 	} else if (tokencmp(tokens[1], "all")) {
-		state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
-		for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
-			int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, true, 0)));
-			if (!checkRequestSent) {
-				result = false;
-				fprintf(stderr, "ERROR: failed to send request to check process `%s'.\n", it->first.toString().c_str());
-			}
-		}
 		if (address_interface->size() == 0) {
 			fprintf(stderr,
 			        "ERROR: no processes to check. You must run the `expensive_data_check’ "
 			        "command before running `expensive_data_check all’.\n");
 		} else {
-			printf("Attempted to kill and check %zu processes\n", address_interface->size());
+			std::vector<std::string> addressesVec;
+			for (const auto& [address, _] : *address_interface) {
+				addressesVec.push_back(address.toString());
+			}
+			addressesStr = boost::algorithm::join(addressesVec, ",");
+			// make sure we only call the interface once to send requests in parallel
+			int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
+			if (!checkRequestsSent) {
+				result = false;
+				fprintf(stderr,
+				        "ERROR: failed to send requests to check all processes, please run the `expensive_data_check’ "
+				        "command again to fetch latest addresses.\n");
+			} else {
+				printf("Attempted to kill and check %zu processes\n", address_interface->size());
+			}
 		}
 	} else {
 		state int i;
@@ -83,15 +93,21 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 		}
 
 		if (result) {
+			std::vector<std::string> addressesVec;
 			for (i = 1; i < tokens.size(); i++) {
-				int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], true, 0)));
+				addressesVec.push_back(tokens[i].toString());
-				if (!checkRequestSent) {
+			}
-					result = false;
+			addressesStr = boost::algorithm::join(addressesVec, ",");
-					fprintf(
+			int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
-					    stderr, "ERROR: failed to send request to check process `%s'.\n", tokens[i].toString().c_str());
+			if (!checkRequestsSent) {
-				}
+				result = false;
+				fprintf(stderr,
+				        "ERROR: failed to send requests to check processes `%s', please run the `expensive_data_check’ "
+				        "command again to fetch latest addresses.\n",
+				        addressesStr.c_str());
+			} else {
+				printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
 			}
-			printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
 		}
 	}
 	return result;

fdbcli/KillCommand.actor.cpp

@@ -18,6 +18,8 @@
  * limitations under the License.
  */
 
+#include "boost/algorithm/string.hpp"
+
 #include "fdbcli/fdbcli.actor.h"
 
 #include "fdbclient/FDBOptions.g.h"
@@ -37,8 +39,10 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
                                     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
 	ASSERT(tokens.size() >= 1);
 	state bool result = true;
+	state std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
+		address_interface->clear();
 		wait(getWorkerInterfaces(tr, address_interface));
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
@@ -54,21 +58,27 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 		}
 		printf("\n");
 	} else if (tokencmp(tokens[1], "all")) {
-		state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
-		for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
-			int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, false, 0)));
-			if (!killRequestSent) {
-				result = false;
-				fprintf(stderr, "ERROR: failed to send request to kill process `%s'.\n", it->first.toString().c_str());
-			}
-		}
 		if (address_interface->size() == 0) {
 			result = false;
 			fprintf(stderr,
 			        "ERROR: no processes to kill. You must run the `kill’ command before "
 			        "running `kill all’.\n");
 		} else {
-			printf("Attempted to kill %zu processes\n", address_interface->size());
+			std::vector<std::string> addressesVec;
+			for (const auto& [address, _] : *address_interface) {
+				addressesVec.push_back(address.toString());
+			}
+			addressesStr = boost::algorithm::join(addressesVec, ",");
+			// make sure we only call the interface once to send requests in parallel
+			int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
+			if (!killRequestsSent) {
+				result = false;
+				fprintf(stderr,
+				        "ERROR: failed to send requests to all processes, please run the `kill’ command again to fetch "
+				        "latest addresses.\n");
+			} else {
+				printf("Attempted to kill %zu processes\n", address_interface->size());
+			}
 		}
 	} else {
 		state int i;
@@ -81,15 +91,21 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 		}
 
 		if (result) {
+			std::vector<std::string> addressesVec;
 			for (i = 1; i < tokens.size(); i++) {
-				int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, 0)));
+				addressesVec.push_back(tokens[i].toString());
-				if (!killRequestSent) {
+			}
-					result = false;
+			addressesStr = boost::algorithm::join(addressesVec, ",");
-					fprintf(
+			int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
-					    stderr, "ERROR: failed to send request to kill process `%s'.\n", tokens[i].toString().c_str());
+			if (!killRequestsSent) {
-				}
+				result = false;
+				fprintf(stderr,
+				        "ERROR: failed to send requests to kill processes `%s', please run the `kill’ command again to "
+				        "fetch latest addresses.\n",
+				        addressesStr.c_str());
+			} else {
+				printf("Attempted to kill %zu processes\n", tokens.size() - 1);
 			}
-			printf("Attempted to kill %zu processes\n", tokens.size() - 1);
 		}
 	}
 	return result;

fdbcli/ProfileCommand.actor.cpp

@@ -35,7 +35,10 @@
 
 namespace fdb_cli {
 
-ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans) {
+ACTOR Future<bool> profileCommandActor(Database db,
+                                       Reference<ITransaction> tr,
+                                       std::vector<StringRef> tokens,
+                                       bool intrans) {
 	state bool result = true;
 	if (tokens.size() == 1) {
 		printUsage(tokens[0]);
@@ -45,7 +48,7 @@ ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<S
 			fprintf(stderr, "ERROR: Usage: profile client <get|set>\n");
 			return false;
 		}
-		wait(GlobalConfig::globalConfig().onInitialized());
+		wait(db->globalConfig->onInitialized());
 		if (tokencmp(tokens[2], "get")) {
 			if (tokens.size() != 3) {
 				fprintf(stderr, "ERROR: Addtional arguments to `get` are not supported.\n");
@@ -53,12 +56,12 @@ ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<S
 			}
 			std::string sampleRateStr = "default";
 			std::string sizeLimitStr = "default";
-			const double sampleRateDbl = GlobalConfig::globalConfig().get<double>(
+			const double sampleRateDbl =
-			    fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
+			    db->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
 			if (!std::isinf(sampleRateDbl)) {
 				sampleRateStr = std::to_string(sampleRateDbl);
 			}
-			const int64_t sizeLimit = GlobalConfig::globalConfig().get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
+			const int64_t sizeLimit = db->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
 			if (sizeLimit != -1) {
 				sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
 			}

fdbcli/SuspendCommand.actor.cpp

@@ -18,6 +18,8 @@
  * limitations under the License.
  */
 
+#include "boost/algorithm/string.hpp"
+
 #include "fdbcli/fdbcli.actor.h"
 
 #include "fdbclient/FDBOptions.g.h"
@@ -37,8 +39,10 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
                                        std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
 	ASSERT(tokens.size() >= 1);
 	state bool result = true;
+	state std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
+		address_interface->clear();
 		wait(getWorkerInterfaces(tr, address_interface));
 		if (address_interface->size() == 0) {
 			printf("\nNo addresses can be suspended.\n");
@@ -72,19 +76,23 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 				printUsage(tokens[0]);
 				result = false;
 			} else {
-				int64_t timeout_ms = seconds * 1000;
+				std::vector<std::string> addressesVec;
-				tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&timeout_ms, sizeof(int64_t)));
 				for (i = 2; i < tokens.size(); i++) {
-					int64_t suspendRequestSent =
+					addressesVec.push_back(tokens[i].toString());
-					    wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, static_cast<int>(seconds))));
+				}
-					if (!suspendRequestSent) {
+				addressesStr = boost::algorithm::join(addressesVec, ",");
-						result = false;
+				int64_t suspendRequestSent =
-						fprintf(stderr,
+				    wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast<int>(seconds))));
-						        "ERROR: failed to send request to suspend process `%s'.\n",
+				if (!suspendRequestSent) {
-						        tokens[i].toString().c_str());
+					result = false;
-					}
+					fprintf(
+					    stderr,
+					    "ERROR: failed to send requests to suspend processes `%s', please run the `suspend’ command "
+					    "to fetch latest addresses.\n",
+					    addressesStr.c_str());
+				} else {
+					printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
 				}
-				printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
 			}
 		}
 	}

fdbcli/fdbcli.actor.cpp

@@ -782,7 +782,7 @@ void fdbcliCompCmd(std::string const& text, std::vector<std::string>& lc) {
 	int count = tokens.size();
 
 	// for(int i = 0; i < count; i++) {
-	// 	printf("Token (%d): `%s'\n", i, tokens[i].toString().c_str());
+	//	printf("Token (%d): `%s'\n", i, tokens[i].toString().c_str());
 	// }
 
 	std::string ntext = "";
@@ -1012,6 +1012,36 @@ Future<T> stopNetworkAfter(Future<T> what) {
 	}
 }
 
+ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+                                Reference<FlowLock> connectLock,
+                                KeyValue kv) {
+	wait(connectLock->take());
+	state FlowLock::Releaser releaser(*connectLock);
+	state ClientWorkerInterface workerInterf =
+	    BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
+	state ClientLeaderRegInterface leaderInterf(workerInterf.address());
+	choose {
+		when(Optional<LeaderInfo> rep =
+		         wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
+			StringRef ip_port =
+			    (kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
+			        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
+			(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
+
+			if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+				Key full_ip_port2 =
+				    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
+				StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
+				                         ? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
+				                         : full_ip_port2;
+				(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
+			}
+		}
+		when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
+	}
+	return Void();
+}
+
 ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	state LineNoise& linenoise = *plinenoise;
 	state bool intrans = false;
@@ -1552,7 +1582,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 
 				if (tokencmp(tokens[0], "profile")) {
 					getTransaction(db, managementTenant, tr, options, intrans);
-					bool _result = wait(makeInterruptable(profileCommandActor(tr, tokens, intrans)));
+					bool _result = wait(makeInterruptable(profileCommandActor(localDb, tr, tokens, intrans)));
 					if (!_result)
 						is_error = true;
 					continue;

fdbcli/fdbcli.actor.h

@@ -121,10 +121,7 @@ extern const KeyRangeRef processClassTypeSpecialKeyRange;
 // Other special keys
 inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_message");
 // help functions (Copied from fdbcli.actor.cpp)
-// decode worker interfaces
+
-ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                Reference<FlowLock> connectLock,
-                                KeyValue kv);
 // get all workers' info
 ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers);
 
@@ -217,7 +214,10 @@ ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
                                     bool clearSSFailureZoneString = false);
 ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // profile command
-ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans);
+ACTOR Future<bool> profileCommandActor(Database db,
+                                       Reference<ITransaction> tr,
+                                       std::vector<StringRef> tokens,
+                                       bool intrans);
 // setclass command
 ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // snapshot command

fdbclient/DatabaseContext.h

@@ -222,6 +222,8 @@ struct KeyRangeLocationInfo {
 	  : tenantEntry(tenantEntry), range(range), locations(locations) {}
 };
 
+class GlobalConfig;
+
 class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAllocated<DatabaseContext>, NonCopyable {
 public:
 	static DatabaseContext* allocateOnForeignThread() {
@@ -627,6 +629,7 @@ public:
 	using TransactionT = ReadYourWritesTransaction;
 	Reference<TransactionT> createTransaction();
 
+	std::unique_ptr<GlobalConfig> globalConfig;
 	EventCacheHolder connectToDatabaseEventCacheHolder;
 
 private:

fdbclient/FDBTypes.h

@@ -148,6 +148,11 @@ static const Tag invalidTag{ tagLocalitySpecial, 0 };
 static const Tag txsTag{ tagLocalitySpecial, 1 };
 static const Tag cacheTag{ tagLocalitySpecial, 2 };
 
+const int MATCH_INDEX_ALL = 0;
+const int MATCH_INDEX_NONE = 1;
+const int MATCH_INDEX_MATCHED_ONLY = 2;
+const int MATCH_INDEX_UNMATCHED_ONLY = 3;
+
 enum { txsTagOld = -1, invalidTagOld = -100 };
 
 struct TagsAndMessage {
@@ -759,9 +764,18 @@ struct MappedKeyValueRef : KeyValueRef {
 
 	MappedReqAndResultRef reqAndResult;
 
+	// boundary KVs are always returned so that caller can use it as a continuation,
+	// for non-boundary KV, it is always false.
+	// for boundary KV, it is true only when the secondary query succeeds(return non-empty).
+	// Note: only MATCH_INDEX_MATCHED_ONLY and MATCH_INDEX_UNMATCHED_ONLY modes can make use of it,
+	// to decide whether the boudnary is a match/unmatch.
+	// In the case of MATCH_INDEX_ALL and MATCH_INDEX_NONE, caller should not care if boundary has a match or not.
+	bool boundaryAndExist;
+
 	MappedKeyValueRef() = default;
 	MappedKeyValueRef(Arena& a, const MappedKeyValueRef& copyFrom) : KeyValueRef(a, copyFrom) {
 		const auto& reqAndResultCopyFrom = copyFrom.reqAndResult;
+		boundaryAndExist = copyFrom.boundaryAndExist;
 		if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResultCopyFrom)) {
 			auto getValue = std::get<GetValueReqAndResultRef>(reqAndResultCopyFrom);
 			reqAndResult = GetValueReqAndResultRef(a, getValue);
@@ -775,7 +789,7 @@ struct MappedKeyValueRef : KeyValueRef {
 
 	bool operator==(const MappedKeyValueRef& rhs) const {
 		return static_cast<const KeyValueRef&>(*this) == static_cast<const KeyValueRef&>(rhs) &&
-		       reqAndResult == rhs.reqAndResult;
+		       reqAndResult == rhs.reqAndResult && boundaryAndExist == rhs.boundaryAndExist;
 	}
 	bool operator!=(const MappedKeyValueRef& rhs) const { return !(rhs == *this); }
 
@@ -785,7 +799,7 @@ struct MappedKeyValueRef : KeyValueRef {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, ((KeyValueRef&)*this), reqAndResult);
+		serializer(ar, ((KeyValueRef&)*this), reqAndResult, boundaryAndExist);
 	}
 };
 

fdbclient/GlobalConfig.actor.cpp

@@ -37,12 +37,33 @@ const KeyRef transactionTagSampleCost = LiteralStringRef("config/transaction_tag
 const KeyRef samplingFrequency = LiteralStringRef("visibility/sampling/frequency");
 const KeyRef samplingWindow = LiteralStringRef("visibility/sampling/window");
 
-GlobalConfig::GlobalConfig(Database& cx) : cx(cx), lastUpdate(0) {}
+GlobalConfig::GlobalConfig(const Database& cx) : cx(cx), lastUpdate(0) {}
 
-GlobalConfig& GlobalConfig::globalConfig() {
+void GlobalConfig::applyChanges(Transaction& tr,
-	void* res = g_network->global(INetwork::enGlobalConfig);
+                                const VectorRef<KeyValueRef>& insertions,
-	ASSERT(res);
+                                const VectorRef<KeyRangeRef>& clears) {
-	return *reinterpret_cast<GlobalConfig*>(res);
+	VersionHistory vh{ 0 };
+	for (const auto& kv : insertions) {
+		vh.mutations.emplace_back_deep(vh.mutations.arena(), MutationRef(MutationRef::SetValue, kv.key, kv.value));
+		tr.set(kv.key.withPrefix(globalConfigKeysPrefix), kv.value);
+	}
+	for (const auto& range : clears) {
+		vh.mutations.emplace_back_deep(vh.mutations.arena(),
+		                               MutationRef(MutationRef::ClearRange, range.begin, range.end));
+		tr.clear(
+		    KeyRangeRef(range.begin.withPrefix(globalConfigKeysPrefix), range.end.withPrefix(globalConfigKeysPrefix)));
+	}
+
+	// Record the mutations in this commit into the global configuration history.
+	Key historyKey = addVersionStampAtEnd(globalConfigHistoryPrefix);
+	ObjectWriter historyWriter(IncludeVersion());
+	historyWriter.serialize(vh);
+	tr.atomicOp(historyKey, historyWriter.toStringRef(), MutationRef::SetVersionstampedKey);
+
+	// Write version key to trigger update in cluster controller.
+	tr.atomicOp(globalConfigVersionKey,
+	            LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp
+	            MutationRef::SetVersionstampedValue);
 }
 
 Key GlobalConfig::prefixedKey(KeyRef key) {

fdbclient/GlobalConfig.actor.h

@@ -33,6 +33,7 @@
 #include <unordered_map>
 
 #include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.h"
 #include "fdbclient/ReadYourWrites.h"
 
@@ -66,34 +67,32 @@ struct ConfigValue : ReferenceCounted<ConfigValue> {
 
 class GlobalConfig : NonCopyable {
 public:
-	// Creates a GlobalConfig singleton, accessed by calling
+	// Requires a database object to allow global configuration to run
-	// GlobalConfig::globalConfig(). This function requires a database object
+	// transactions on the database.
-	// to allow global configuration to run transactions on the database, and
+	explicit GlobalConfig(const Database& cx);
-	// an AsyncVar object to watch for changes on. The ClientDBInfo pointer
+
+	// Requires an AsyncVar object to watch for changes on. The ClientDBInfo pointer
 	// should point to a ClientDBInfo object which will contain the updated
 	// global configuration history when the given AsyncVar changes. This
 	// function should be called whenever the database object changes, in order
 	// to allow global configuration to run transactions on the latest
 	// database.
 	template <class T>
-	static void create(Database& cx, Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
+	void init(Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
-		if (g_network->global(INetwork::enGlobalConfig) == nullptr) {
+		_updater = updater(this, dbInfo);
-			auto config = new GlobalConfig{ cx };
+		// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
-			g_network->setGlobal(INetwork::enGlobalConfig, config);
+		// TODO: Change AsyncTrigger to a Reference
-			config->_updater = updater(config, dbInfo);
+		_forward = forward(db, std::addressof(dbInfoChanged));
-			// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
-			// TODO: Change AsyncTrigger to a Reference
-			forward(db, std::addressof(config->dbInfoChanged));
-		} else {
-			GlobalConfig* config = reinterpret_cast<GlobalConfig*>(g_network->global(INetwork::enGlobalConfig));
-			config->cx = cx;
-		}
 	}
 
-	// Returns a reference to the global GlobalConfig object. Clients should
+	// Given a list of insertions and clears, applies the necessary changes to
-	// call this function whenever they need to read a value out of the global
+	// the given transaction to update the global configuration database. Keys
-	// configuration.
+	// in the list of mutations should not include the global configuration
-	static GlobalConfig& globalConfig();
+	// prefix (`\xff\xff/global_config/`). The caller must still commit the
+	// given transaction in order to persist the changes.
+	static void applyChanges(Transaction& tr,
+	                         const VectorRef<KeyValueRef>& insertions,
+	                         const VectorRef<KeyRangeRef>& clears);
 
 	// Use this function to turn a global configuration key defined above into
 	// the full path needed to set the value in the database.
@@ -150,8 +149,6 @@ public:
 	void trigger(KeyRef key, std::function<void(std::optional<std::any>)> fn);
 
 private:
-	GlobalConfig(Database& cx);
-
 	// The functions below only affect the local copy of the global
 	// configuration keyspace! To insert or remove values across all nodes you
 	// must use a transaction (see the note above).
@@ -173,6 +170,7 @@ private:
 
 	Database cx;
 	AsyncTrigger dbInfoChanged;
+	Future<Void> _forward;
 	Future<Void> _updater;
 	Promise<Void> initialized;
 	AsyncTrigger configChanged;

fdbclient/IClientApi.h

@@ -68,6 +68,7 @@ public:
 	                                                       const KeySelectorRef& end,
 	                                                       const StringRef& mapper,
 	                                                       GetRangeLimits limits,
+	                                                       int matchIndex = MATCH_INDEX_ALL,
 	                                                       bool snapshot = false,
 	                                                       bool reverse = false) = 0;
 	virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
@@ -152,7 +153,11 @@ public:
 	virtual void addref() = 0;
 	virtual void delref() = 0;
 
-	// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
+	// Management API, attempt to kill or suspend a process, return 1 for request being sent out, 0 for failure
+	// The address string can be extended to a comma-delimited string like <addr1>,<addr2>...,<addrN> to send reboot
+	// requests to multiple processes simultaneously
+	// If multiple addresses are provided, it returns 1 for requests being sent out to all provided addresses.
+	// On the contrary, if the client cannot connect to any of the given address, no requests will be sent out
 	virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
 	// Management API, force the database to recover into DCID, causing the database to lose the most recently committed
 	// mutations

fdbclient/ISingleThreadTransaction.h

@@ -74,6 +74,7 @@ public:
 	                                                 KeySelector end,
 	                                                 Key mapper,
 	                                                 GetRangeLimits limits,
+	                                                 int matchIndex = MATCH_INDEX_ALL,
 	                                                 Snapshot = Snapshot::False,
 	                                                 Reverse = Reverse::False) = 0;
 	virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;

fdbclient/MonitorLeader.actor.cpp

@@ -270,6 +270,9 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString/hostname") {
 
 ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnectionString* self) {
 	state std::set<NetworkAddress> allCoordinatorsSet;
+	for (const auto& coord : self->coords) {
+		allCoordinatorsSet.insert(coord);
+	}
 	std::vector<Future<Void>> fs;
 	for (auto& hostname : self->hostnames) {
 		fs.push_back(map(hostname.resolve(), [&](Optional<NetworkAddress> const& addr) -> Void {
@@ -280,9 +283,6 @@ ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnect
 		}));
 	}
 	wait(waitForAll(fs));
-	for (const auto& coord : self->coords) {
-		allCoordinatorsSet.insert(coord);
-	}
 	std::vector<NetworkAddress> allCoordinators(allCoordinatorsSet.begin(), allCoordinatorsSet.end());
 	std::sort(allCoordinators.begin(), allCoordinators.end());
 	return allCoordinators;
@@ -300,7 +300,7 @@ TEST_CASE("/fdbclient/MonitorLeader/PartialResolve") {
 
 	INetworkConnections::net()->addMockTCPEndpoint(hn, port, { address });
 
-	state ClusterConnectionString cs(connectionString);
+	ClusterConnectionString cs(connectionString);
 	state std::vector<NetworkAddress> allCoordinators = wait(cs.tryResolveHostnames());
 	ASSERT(allCoordinators.size() == 1 &&
 	       std::find(allCoordinators.begin(), allCoordinators.end(), address) != allCoordinators.end());
@@ -890,8 +890,6 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		state ClientLeaderRegInterface clientLeaderServer = clientLeaderServers[index];
 		state OpenDatabaseCoordRequest req;
 
-		coordinator->set(clientLeaderServer);
-
 		req.clusterKey = cs.clusterKey();
 		req.hostnames = cs.hostnames;
 		req.coordinators = cs.coordinators();
@@ -922,16 +920,26 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			incorrectTime = Optional<double>();
 		}
 
-		state ErrorOr<CachedSerialization<ClientDBInfo>> rep;
+		state Future<ErrorOr<CachedSerialization<ClientDBInfo>>> repFuture;
 		if (clientLeaderServer.hostname.present()) {
-			wait(store(rep,
+			repFuture = tryGetReplyFromHostname(req,
-			           tryGetReplyFromHostname(req,
+			                                    clientLeaderServer.hostname.get(),
-			                                   clientLeaderServer.hostname.get(),
+			                                    WLTOKEN_CLIENTLEADERREG_OPENDATABASE,
-			                                   WLTOKEN_CLIENTLEADERREG_OPENDATABASE,
+			                                    TaskPriority::CoordinationReply);
-			                                   TaskPriority::CoordinationReply)));
 		} else {
-			wait(store(rep, clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply)));
+			repFuture = clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply);
 		}
+
+		// We need to update the coordinator even if it hasn't changed in case we are establishing a new connection in
+		// FlowTransport. If so, setting the coordinator here forces protocol version monitoring to restart with the new
+		// peer object.
+		//
+		// Both the tryGetReply call and the creation of the ClientLeaderRegInterface above should result in the Peer
+		// object being created in FlowTransport. Having this peer is a prerequisite to us signaling the AsyncVar.
+		coordinator->setUnconditional(clientLeaderServer);
+
+		state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait(repFuture);
+
 		if (rep.present()) {
 			if (rep.get().read().forward.present()) {
 				TraceEvent("MonitorProxiesForwarding")

fdbclient/MultiVersionTransaction.actor.cpp

@@ -158,6 +158,7 @@ ThreadFuture<MappedRangeResult> DLTransaction::getMappedRange(const KeySelectorR
                                                               const KeySelectorRef& end,
                                                               const StringRef& mapper,
                                                               GetRangeLimits limits,
+                                                              int matchIndex,
                                                               bool snapshot,
                                                               bool reverse) {
 	FdbCApi::FDBFuture* f = api->transactionGetMappedRange(tr,
@@ -175,6 +176,7 @@ ThreadFuture<MappedRangeResult> DLTransaction::getMappedRange(const KeySelectorR
 	                                                       limits.bytes,
 	                                                       FDB_STREAMING_MODE_EXACT,
 	                                                       0,
+	                                                       matchIndex,
 	                                                       snapshot,
 	                                                       reverse);
 	return toThreadFuture<MappedRangeResult>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
@@ -971,10 +973,11 @@ ThreadFuture<MappedRangeResult> MultiVersionTransaction::getMappedRange(const Ke
                                                                         const KeySelectorRef& end,
                                                                         const StringRef& mapper,
                                                                         GetRangeLimits limits,
+                                                                        int matchIndex,
                                                                         bool snapshot,
                                                                         bool reverse) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getMappedRange(begin, end, mapper, limits, snapshot, reverse)
+	auto f = tr.transaction ? tr.transaction->getMappedRange(begin, end, mapper, limits, matchIndex, snapshot, reverse)
 	                        : makeTimeout<MappedRangeResult>();
 	return abortableFuture(f, tr.onChange);
 }
@@ -1609,7 +1612,7 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
 		// When the protocol version changes, clear the corresponding entry in the shared state map
 		// so it can be re-initialized. Only do so if there was a valid previous protocol version.
 		if (dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
-			MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterFilePath);
+			MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterFilePath, dbProtocolVersion.get());
 		}
 
 		dbProtocolVersion = protocolVersion;
@@ -1722,8 +1725,10 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 		}
 	}
 	if (db.isValid() && dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
-		auto updateResult = MultiVersionApi::api->updateClusterSharedStateMap(clusterFilePath, db);
+		auto updateResult =
+		    MultiVersionApi::api->updateClusterSharedStateMap(clusterFilePath, dbProtocolVersion.get(), db);
 		auto handler = mapThreadFuture<Void, Void>(updateResult, [this](ErrorOr<Void> result) {
+			TraceEvent("ClusterSharedStateUpdated").detail("ClusterFilePath", clusterFilePath);
 			dbVar->set(db);
 			return ErrorOr<Void>(Void());
 		});
@@ -2389,12 +2394,30 @@ void MultiVersionApi::updateSupportedVersions() {
 	}
 }
 
-ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db) {
+ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clusterFilePath,
+                                                                ProtocolVersion dbProtocolVersion,
+                                                                Reference<IDatabase> db) {
 	MutexHolder holder(lock);
 	if (clusterSharedStateMap.find(clusterFilePath) == clusterSharedStateMap.end()) {
-		clusterSharedStateMap[clusterFilePath] = db->createSharedState();
+		TraceEvent("CreatingClusterSharedState")
+		    .detail("ClusterFilePath", clusterFilePath)
+		    .detail("ProtocolVersion", dbProtocolVersion);
+		clusterSharedStateMap[clusterFilePath] = { db->createSharedState(), dbProtocolVersion };
 	} else {
-		ThreadFuture<DatabaseSharedState*> entry = clusterSharedStateMap[clusterFilePath];
+		auto& sharedStateInfo = clusterSharedStateMap[clusterFilePath];
+		if (sharedStateInfo.protocolVersion != dbProtocolVersion) {
+			// This situation should never happen, because we are connecting to the same cluster,
+			// so the protocol version must be the same
+			TraceEvent(SevError, "ClusterStateProtocolVersionMismatch")
+			    .detail("ClusterFilePath", clusterFilePath)
+			    .detail("ProtocolVersionExpected", dbProtocolVersion)
+			    .detail("ProtocolVersionFound", sharedStateInfo.protocolVersion);
+			return Void();
+		}
+		TraceEvent("SettingClusterSharedState")
+		    .detail("ClusterFilePath", clusterFilePath)
+		    .detail("ProtocolVersion", dbProtocolVersion);
+		ThreadFuture<DatabaseSharedState*> entry = sharedStateInfo.sharedStateFuture;
 		return mapThreadFuture<DatabaseSharedState*, Void>(entry, [db](ErrorOr<DatabaseSharedState*> result) {
 			if (result.isError()) {
 				return ErrorOr<Void>(result.getError());
@@ -2407,16 +2430,29 @@ ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clus
 	return Void();
 }
 
-void MultiVersionApi::clearClusterSharedStateMapEntry(std::string clusterFilePath) {
+void MultiVersionApi::clearClusterSharedStateMapEntry(std::string clusterFilePath, ProtocolVersion dbProtocolVersion) {
 	MutexHolder holder(lock);
 	auto mapEntry = clusterSharedStateMap.find(clusterFilePath);
+	// It can be that other database instances on the same cluster path are already upgraded and thus
+	// have cleared or even created a new shared object entry
 	if (mapEntry == clusterSharedStateMap.end()) {
-		TraceEvent(SevError, "ClusterSharedStateMapEntryNotFound").detail("ClusterFilePath", clusterFilePath);
+		TraceEvent("ClusterSharedStateMapEntryNotFound").detail("ClusterFilePath", clusterFilePath);
 		return;
 	}
-	auto ssPtr = mapEntry->second.get();
+	auto sharedStateInfo = mapEntry->second;
+	if (sharedStateInfo.protocolVersion != dbProtocolVersion) {
+		TraceEvent("ClusterSharedStateClearSkipped")
+		    .detail("ClusterFilePath", clusterFilePath)
+		    .detail("ProtocolVersionExpected", dbProtocolVersion)
+		    .detail("ProtocolVersionFound", sharedStateInfo.protocolVersion);
+		return;
+	}
+	auto ssPtr = sharedStateInfo.sharedStateFuture.get();
 	ssPtr->delRef(ssPtr);
 	clusterSharedStateMap.erase(mapEntry);
+	TraceEvent("ClusterSharedStateCleared")
+	    .detail("ClusterFilePath", clusterFilePath)
+	    .detail("ProtocolVersion", dbProtocolVersion);
 }
 
 std::vector<std::string> parseOptionValues(std::string valueStr) {

fdbclient/MultiVersionTransaction.h

@@ -80,6 +80,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 		 * and take the shortcut. */
 		FDBGetRangeReqAndResult getRange;
 		unsigned char buffer[32];
+		bool boundaryAndExist;
 	} FDBMappedKeyValue;
 
 #pragma pack(push, 4)
@@ -218,6 +219,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                        int targetBytes,
 	                                        FDBStreamingMode mode,
 	                                        int iteration,
+	                                        int matchIndex,
 	                                        fdb_bool_t snapshot,
 	                                        fdb_bool_t reverse);
 	FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr);
@@ -349,6 +351,7 @@ public:
 	                                               const KeySelectorRef& end,
 	                                               const StringRef& mapper,
 	                                               GetRangeLimits limits,
+	                                               int matchIndex,
 	                                               bool snapshot,
 	                                               bool reverse) override;
 	ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
@@ -537,6 +540,7 @@ public:
 	                                               const KeySelectorRef& end,
 	                                               const StringRef& mapper,
 	                                               GetRangeLimits limits,
+	                                               int matchIndex,
 	                                               bool snapshot,
 	                                               bool reverse) override;
 	ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
@@ -861,8 +865,10 @@ public:
 
 	bool callbackOnMainThread;
 	bool localClientDisabled;
-	ThreadFuture<Void> updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db);
+	ThreadFuture<Void> updateClusterSharedStateMap(std::string clusterFilePath,
-	void clearClusterSharedStateMapEntry(std::string clusterFilePath);
+	                                               ProtocolVersion dbProtocolVersion,
+	                                               Reference<IDatabase> db);
+	void clearClusterSharedStateMapEntry(std::string clusterFilePath, ProtocolVersion dbProtocolVersion);
 
 	static bool apiVersionAtLeast(int minVersion);
 
@@ -888,7 +894,11 @@ private:
 	std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
 	// Map of clusterFilePath -> DatabaseSharedState pointer Future
 	// Upon cluster version upgrade, clear the map entry for that cluster
-	std::map<std::string, ThreadFuture<DatabaseSharedState*>> clusterSharedStateMap;
+	struct SharedStateInfo {
+		ThreadFuture<DatabaseSharedState*> sharedStateFuture;
+		ProtocolVersion protocolVersion;
+	};
+	std::map<std::string, SharedStateInfo> clusterSharedStateMap;
 
 	bool networkStartSetup;
 	volatile bool networkSetup;

fdbclient/NativeAPI.actor.cpp

@@ -23,12 +23,14 @@
 #include <algorithm>
 #include <cstdio>
 #include <iterator>
+#include <memory>
 #include <regex>
 #include <unordered_set>
 #include <tuple>
 #include <utility>
 #include <vector>
 
+#include "boost/algorithm/string.hpp"
 #include "contrib/fmt-8.1.1/include/fmt/format.h"
 
 #include "fdbclient/FDBOptions.g.h"
@@ -809,12 +811,12 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 				}
 			}
 			cx->clientStatusUpdater.outStatusQ.clear();
-			wait(GlobalConfig::globalConfig().onInitialized());
+			wait(cx->globalConfig->onInitialized());
-			double sampleRate = GlobalConfig::globalConfig().get<double>(fdbClientInfoTxnSampleRate,
+			double sampleRate =
-			                                                             std::numeric_limits<double>::infinity());
+			    cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
 			double clientSamplingProbability =
 			    std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate;
-			int64_t sizeLimit = GlobalConfig::globalConfig().get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
+			int64_t sizeLimit = cx->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
 			int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit;
 			if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability)
 				wait(delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit));
@@ -1481,6 +1483,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 	cacheListMonitor = monitorCacheList(this);
 
 	smoothMidShardSize.reset(CLIENT_KNOBS->INIT_MID_SHARD_BYTES);
+	globalConfig = std::make_unique<GlobalConfig>(Database(this));
 
 	if (apiVersionAtLeast(710)) {
 		registerSpecialKeysImpl(
@@ -1937,13 +1940,12 @@ Future<Void> DatabaseContext::onProxiesChanged() const {
 }
 
 bool DatabaseContext::sampleReadTags() const {
-	double sampleRate = GlobalConfig::globalConfig().get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
+	double sampleRate = globalConfig->get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
 	return sampleRate > 0 && deterministicRandom()->random01() <= sampleRate;
 }
 
 bool DatabaseContext::sampleOnCost(uint64_t cost) const {
-	double sampleCost =
+	double sampleCost = globalConfig->get<double>(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST);
-	    GlobalConfig::globalConfig().get<double>(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST);
 	if (sampleCost <= 0)
 		return false;
 	return deterministicRandom()->random01() <= (double)cost / sampleCost;
@@ -2219,10 +2221,10 @@ Database Database::createDatabase(Reference<IClusterConnectionRecord> connRecord
 	}
 
 	auto database = Database(db);
-	GlobalConfig::create(
+	database->globalConfig->init(Reference<AsyncVar<ClientDBInfo> const>(clientInfo),
-	    database, Reference<AsyncVar<ClientDBInfo> const>(clientInfo), std::addressof(clientInfo->get()));
+	                             std::addressof(clientInfo->get()));
-	GlobalConfig::globalConfig().trigger(samplingFrequency, samplingProfilerUpdateFrequency);
+	database->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
-	GlobalConfig::globalConfig().trigger(samplingWindow, samplingProfilerUpdateWindow);
+	database->globalConfig->trigger(samplingWindow, samplingProfilerUpdateWindow);
 
 	TraceEvent("ConnectToDatabase", database->dbId)
 	    .detail("Version", FDB_VT_VERSION)
@@ -3790,12 +3792,24 @@ PublicRequestStream<GetKeyValuesFamilyRequest> StorageServerInterface::*getRange
 	}
 }
 
+template <class GetKeyValuesFamilyRequest>
+void setMatchIndex(GetKeyValuesFamilyRequest& req, int matchIndex) {
+	if constexpr (std::is_same<GetKeyValuesFamilyRequest, GetKeyValuesRequest>::value) {
+		// do nothing;
+	} else if (std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value) {
+		req.matchIndex = matchIndex;
+	} else {
+		UNREACHABLE();
+	}
+}
+
 ACTOR template <class GetKeyValuesFamilyRequest, class GetKeyValuesFamilyReply, class RangeResultFamily>
 Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
                                         Version version,
                                         KeyRange keys,
                                         Key mapper,
                                         GetRangeLimits limits,
+                                        int matchIndex,
                                         Reverse reverse,
                                         UseTenant useTenant) {
 	state RangeResultFamily output;
@@ -3829,6 +3843,7 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 			req.version = version;
 			req.begin = firstGreaterOrEqual(range.begin);
 			req.end = firstGreaterOrEqual(range.end);
+			setMatchIndex<GetKeyValuesFamilyRequest>(req, matchIndex);
 			req.spanContext = span.context;
 			trState->cx->getLatestCommitVersions(
 			    locations[shard].locations, req.version, trState, req.ssLatestCommitVersions);
@@ -4003,6 +4018,7 @@ Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
                                            KeySelector end,
                                            Key mapper,
                                            GetRangeLimits limits,
+                                           int matchIndex,
                                            Reverse reverse,
                                            UseTenant useTenant) {
 	if (version == latestVersion) {
@@ -4028,7 +4044,7 @@ Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
 	// or allKeys.begin exists in the database/tenant and will be part of the conflict range anyways
 
 	RangeResultFamily _r = wait(getExactRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
-	    trState, version, KeyRangeRef(b, e), mapper, limits, reverse, useTenant));
+	    trState, version, KeyRangeRef(b, e), mapper, limits, matchIndex, reverse, useTenant));
 	RangeResultFamily r = _r;
 
 	if (b == allKeys.begin && ((reverse && !r.more) || !reverse))
@@ -4067,7 +4083,7 @@ int64_t inline getRangeResultFamilyBytes(MappedRangeResultRef result) {
 	int64_t bytes = 0;
 	for (const MappedKeyValueRef& mappedKeyValue : result) {
 		bytes += mappedKeyValue.key.size() + mappedKeyValue.value.size();
-
+		bytes += sizeof(mappedKeyValue.boundaryAndExist);
 		auto& reqAndResult = mappedKeyValue.reqAndResult;
 		if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResult)) {
 			auto getValue = std::get<GetValueReqAndResultRef>(reqAndResult);
@@ -4152,6 +4168,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
                                    Key mapper,
                                    GetRangeLimits limits,
                                    Promise<std::pair<Key, Key>> conflictRange,
+                                   int matchIndex,
                                    Snapshot snapshot,
                                    Reverse reverse,
                                    UseTenant useTenant = UseTenant::True) {
@@ -4204,7 +4221,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 			state GetKeyValuesFamilyRequest req;
 			req.mapper = mapper;
 			req.arena.dependsOn(mapper.arena());
-
+			setMatchIndex<GetKeyValuesFamilyRequest>(req, matchIndex);
 			req.tenantInfo = useTenant ? trState->getTenantInfo() : TenantInfo();
 			req.isFetchKeys = (trState->taskID == TaskPriority::FetchKeys);
 			req.version = readVersion;
@@ -4384,6 +4401,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 						        originalEnd,
 						        mapper,
 						        originalLimits,
+						        matchIndex,
 						        reverse,
 						        useTenant));
 						getRangeFinished(
@@ -4424,6 +4442,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 						        originalEnd,
 						        mapper,
 						        originalLimits,
+						        matchIndex,
 						        reverse,
 						        useTenant));
 						getRangeFinished(
@@ -5009,6 +5028,7 @@ Future<RangeResult> getRange(Reference<TransactionState> const& trState,
 	                                                                     ""_sr,
 	                                                                     limits,
 	                                                                     Promise<std::pair<Key, Key>>(),
+	                                                                     MATCH_INDEX_ALL,
 	                                                                     Snapshot::True,
 	                                                                     reverse,
 	                                                                     useTenant);
@@ -5363,6 +5383,7 @@ Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin
                                                         const KeySelector& end,
                                                         const Key& mapper,
                                                         GetRangeLimits limits,
+                                                        int matchIndex,
                                                         Snapshot snapshot,
                                                         Reverse reverse) {
 	++trState->cx->transactionLogicalReads;
@@ -5405,7 +5426,7 @@ Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin
 	}
 
 	return ::getRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
-	    trState, getReadVersion(), b, e, mapper, limits, conflictRange, snapshot, reverse);
+	    trState, getReadVersion(), b, e, mapper, limits, conflictRange, matchIndex, snapshot, reverse);
 }
 
 Future<RangeResult> Transaction::getRange(const KeySelector& begin,
@@ -5414,17 +5435,18 @@ Future<RangeResult> Transaction::getRange(const KeySelector& begin,
                                           Snapshot snapshot,
                                           Reverse reverse) {
 	return getRangeInternal<GetKeyValuesRequest, GetKeyValuesReply, RangeResult>(
-	    begin, end, ""_sr, limits, snapshot, reverse);
+	    begin, end, ""_sr, limits, MATCH_INDEX_ALL, snapshot, reverse);
 }
 
 Future<MappedRangeResult> Transaction::getMappedRange(const KeySelector& begin,
                                                       const KeySelector& end,
                                                       const Key& mapper,
                                                       GetRangeLimits limits,
+                                                      int matchIndex,
                                                       Snapshot snapshot,
                                                       Reverse reverse) {
 	return getRangeInternal<GetMappedKeyValuesRequest, GetMappedKeyValuesReply, MappedRangeResult>(
-	    begin, end, mapper, limits, snapshot, reverse);
+	    begin, end, mapper, limits, matchIndex, snapshot, reverse);
 }
 
 Future<RangeResult> Transaction::getRange(const KeySelector& begin,
@@ -6962,24 +6984,30 @@ ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
 		coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
 	}
 
-	state Reference<AsyncVar<Optional<ProtocolVersion>> const> protocolVersion =
+	state Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> protocolVersion =
 	    FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress);
 
+	if (!protocolVersion.present()) {
+		TraceEvent(SevWarnAlways, "GetCoordinatorProtocolPeerMissing").detail("Address", coordinatorAddress);
+		wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+		return Optional<ProtocolVersion>();
+	}
+
 	loop {
-		if (protocolVersion->get().present() && protocolVersion->get() != expectedVersion) {
+		if (protocolVersion.get()->get().present() && protocolVersion.get()->get() != expectedVersion) {
-			return protocolVersion->get();
+			return protocolVersion.get()->get();
 		}
 
-		Future<Void> change = protocolVersion->onChange();
+		Future<Void> change = protocolVersion.get()->onChange();
-		if (!protocolVersion->get().present()) {
+		if (!protocolVersion.get()->get().present()) {
 			// If we still don't have any connection info after a timeout, retry sending the protocol version request
 			change = timeout(change, FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Void());
 		}
 
 		wait(change);
 
-		if (!protocolVersion->get().present()) {
+		if (!protocolVersion.get()->get().present()) {
-			return protocolVersion->get();
+			return protocolVersion.get()->get();
 		}
 	}
 }
@@ -7231,10 +7259,10 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 			// condition. Should we abort and wait for the newly split shards to be hot again?
 			state int nLocs = locations.size();
 			// if (nLocs > 1) {
-			// 	TraceEvent("RHDDebug")
+			//	TraceEvent("RHDDebug")
-			// 	    .detail("NumSSIs", nLocs)
+			//	    .detail("NumSSIs", nLocs)
-			// 	    .detail("KeysBegin", keys.begin.printable().c_str())
+			//	    .detail("KeysBegin", keys.begin.printable().c_str())
-			// 	    .detail("KeysEnd", keys.end.printable().c_str());
+			//	    .detail("KeysEnd", keys.end.printable().c_str());
 			// }
 			state std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
 			KeyRef partBegin, partEnd;
@@ -7946,8 +7974,8 @@ void Transaction::checkDeferredError() const {
 
 Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(const Database& cx) {
 	if (!cx->isError()) {
-		double clientSamplingProbability = GlobalConfig::globalConfig().get<double>(
+		double clientSamplingProbability =
-		    fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY);
+		    cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY);
 		if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) &&
 		    deterministicRandom()->random01() < clientSamplingProbability &&
 		    (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) {
@@ -8122,8 +8150,10 @@ ACTOR Future<std::vector<CheckpointMetaData>> getCheckpointMetaData(Database cx,
 
 			futures.clear();
 			for (index = 0; index < locations.size(); ++index) {
-				futures.push_back(getCheckpointMetaDataInternal(
+				futures.push_back(
-				    GetCheckpointRequest(version, keys, format), locations[index].locations, timeout));
+				    getCheckpointMetaDataInternal(GetCheckpointRequest(version, locations[index].range, format),
+				                                  locations[index].locations,
+				                                  timeout));
 				TraceEvent("GetCheckpointShardBegin")
 				    .detail("Range", locations[index].range)
 				    .detail("Version", version)
@@ -8238,57 +8268,74 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 	return (ddCheck && coordinatorCheck);
 }
 
-ACTOR Future<Void> addInterfaceActor(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+// returns true if we can connect to the given worker interface
-                                     Reference<FlowLock> connectLock,
+ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientWorkerInterface workerInterf) {
-                                     KeyValue kv) {
 	wait(connectLock->take());
 	state FlowLock::Releaser releaser(*connectLock);
-	state ClientWorkerInterface workerInterf =
-	    BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
 	state ClientLeaderRegInterface leaderInterf(workerInterf.address());
 	choose {
 		when(Optional<LeaderInfo> rep =
 		         wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
-			StringRef ip_port =
+			return true;
-			    kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key;
+		}
-			(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
+		when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {
-
+			// NOTE : change timeout time here if necessary
-			if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+			return false;
-				Key full_ip_port2 =
-				    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
-				StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
-				                         ? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
-				                         : full_ip_port2;
-				(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
-			}
 		}
-		when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {} // NOTE : change timeout time here if necessary
 	}
-	return Void();
 }
 
 ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
 	// ignore negative value
 	if (duration < 0)
 		duration = 0;
-	// fetch the addresses of all workers
-	state std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
 	if (!cx->getConnectionRecord())
 		return 0;
+	// fetch all workers' addresses and interfaces from CC
 	RangeResult kvs = wait(getWorkerInterfaces(cx->getConnectionRecord()));
 	ASSERT(!kvs.more);
+	// map worker network address to its interface
+	state std::map<Key, ClientWorkerInterface> workerInterfaces;
+	for (const auto& it : kvs) {
+		ClientWorkerInterface workerInterf =
+		    BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
+		Key primaryAddress =
+		    it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls")) : it.key;
+		workerInterfaces[primaryAddress] = workerInterf;
+		// Also add mapping from a worker's second address(if present) to its interface
+		if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+			Key secondAddress =
+			    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
+			secondAddress = secondAddress.endsWith(LiteralStringRef(":tls"))
+			                    ? secondAddress.removeSuffix(LiteralStringRef(":tls"))
+			                    : secondAddress;
+			workerInterfaces[secondAddress] = workerInterf;
+		}
+	}
+	// split and get all the requested addresses to send reboot requests
+	state std::vector<std::string> addressesVec;
+	boost::algorithm::split(addressesVec, addr.toString(), boost::is_any_of(","));
 	// Note: reuse this knob from fdbcli, change it if necessary
 	Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
-	std::vector<Future<Void>> addInterfs;
+	state std::vector<Future<bool>> verifyInterfs;
-	for (const auto& it : kvs) {
+	for (const auto& requestedAddress : addressesVec) {
-		addInterfs.push_back(addInterfaceActor(&address_interface, connectLock, it));
+		// step 1: check that the requested address is in the worker list provided by CC
+		if (!workerInterfaces.count(Key(requestedAddress)))
+			return 0;
+		// step 2: try to establish connections to the requested worker
+		verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterfaces[Key(requestedAddress)]));
+	}
+	// step 3: check if we can establish connections to all requested workers, return if not
+	wait(waitForAll(verifyInterfs));
+	for (const auto& f : verifyInterfs) {
+		if (!f.get())
+			return 0;
+	}
+	// step 4: After verifying we can connect to all requested workers, send reboot requests together
+	for (const auto& address : addressesVec) {
+		// Note: We want to make sure these requests are sent in parallel
+		workerInterfaces[Key(address)].reboot.send(RebootRequest(false, check, duration));
 	}
-	wait(waitForAll(addInterfs));
-	if (!address_interface.count(addr))
-		return 0;
-
-	BinaryReader::fromStringRef<ClientWorkerInterface>(address_interface[addr].first, IncludeVersion())
-	    .reboot.send(RebootRequest(false, check, duration));
 	return 1;
 }
 
@@ -9419,4 +9466,4 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
 
 int64_t getMaxClearKeySize(KeyRef const& key) {
 	return getMaxKeySize(key);
-}
+}

fdbclient/NativeAPI.actor.h

@@ -329,6 +329,7 @@ public:
 	                                                       const KeySelector& end,
 	                                                       const Key& mapper,
 	                                                       GetRangeLimits limits,
+	                                                       int matchIndex = MATCH_INDEX_ALL,
 	                                                       Snapshot = Snapshot::False,
 	                                                       Reverse = Reverse::False);
 
@@ -338,6 +339,7 @@ private:
 	                                           const KeySelector& end,
 	                                           const Key& mapper,
 	                                           GetRangeLimits limits,
+	                                           int matchIndex,
 	                                           Snapshot snapshot,
 	                                           Reverse reverse);
 

fdbclient/PaxosConfigTransaction.h

@@ -54,6 +54,7 @@ public:
 	                                         KeySelector end,
 	                                         Key mapper,
 	                                         GetRangeLimits limits,
+	                                         int matchIndex = MATCH_INDEX_ALL,
 	                                         Snapshot = Snapshot::False,
 	                                         Reverse = Reverse::False) override {
 		throw client_invalid_operation();

fdbclient/ReadYourWrites.actor.cpp

@@ -77,11 +77,12 @@ public:
 
 	template <bool reverse>
 	struct GetMappedRangeReq {
-		GetMappedRangeReq(KeySelector begin, KeySelector end, Key mapper, GetRangeLimits limits)
+		GetMappedRangeReq(KeySelector begin, KeySelector end, Key mapper, int matchIndex, GetRangeLimits limits)
-		  : begin(begin), end(end), mapper(mapper), limits(limits) {}
+		  : begin(begin), end(end), mapper(mapper), limits(limits), matchIndex(matchIndex) {}
 		KeySelector begin, end;
 		Key mapper;
 		GetRangeLimits limits;
+		int matchIndex;
 		using Result = MappedRangeResult;
 	};
 
@@ -1140,9 +1141,13 @@ public:
 			else
 				read.end = KeySelector(firstGreaterOrEqual(key), key.arena());
 		}
-
+		MappedRangeResult v = wait(ryw->tr.getMappedRange(read.begin,
-		MappedRangeResult v = wait(ryw->tr.getMappedRange(
+		                                                  read.end,
-		    read.begin, read.end, read.mapper, read.limits, snapshot, backwards ? Reverse::True : Reverse::False));
+		                                                  read.mapper,
+		                                                  read.limits,
+		                                                  read.matchIndex,
+		                                                  snapshot,
+		                                                  backwards ? Reverse::True : Reverse::False));
 		return v;
 	}
 
@@ -1677,6 +1682,7 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
                                                                     KeySelector end,
                                                                     Key mapper,
                                                                     GetRangeLimits limits,
+                                                                    int matchIndex,
                                                                     Snapshot snapshot,
                                                                     Reverse reverse) {
 	if (getDatabase()->apiVersionAtLeast(630)) {
@@ -1724,9 +1730,9 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
 
 	Future<MappedRangeResult> result =
 	    reverse ? RYWImpl::readWithConflictRangeForGetMappedRange(
-	                  this, RYWImpl::GetMappedRangeReq<true>(begin, end, mapper, limits), snapshot)
+	                  this, RYWImpl::GetMappedRangeReq<true>(begin, end, mapper, matchIndex, limits), snapshot)
 	            : RYWImpl::readWithConflictRangeForGetMappedRange(
-	                  this, RYWImpl::GetMappedRangeReq<false>(begin, end, mapper, limits), snapshot);
+	                  this, RYWImpl::GetMappedRangeReq<false>(begin, end, mapper, matchIndex, limits), snapshot);
 
 	return result;
 }

fdbclient/ReadYourWrites.h

@@ -112,6 +112,7 @@ public:
 	                                         KeySelector end,
 	                                         Key mapper,
 	                                         GetRangeLimits limits,
+	                                         int matchIndex,
 	                                         Snapshot = Snapshot::False,
 	                                         Reverse = Reverse::False) override;
 

fdbclient/S3BlobStore.actor.cpp

@@ -185,11 +185,20 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 
 		Optional<std::string> proxyHost, proxyPort;
 		if (proxy.present()) {
-			if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) {
+			StringRef proxyRef(proxy.get());
-				throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
+			if (proxy.get().find("://") != std::string::npos) {
-				             proxy.get().c_str());
+				StringRef proxyPrefix = proxyRef.eat("://");
+				if (proxyPrefix != "http"_sr) {
+					throw format("Invalid proxy URL prefix '%s'. Either don't use a prefix, or use http://",
+					             proxyPrefix.toString().c_str());
+				}
 			}
-			StringRef p(proxy.get());
+			std::string proxyBody = proxyRef.eat().toString();
+			if (!Hostname::isHostname(proxyBody) && !NetworkAddress::parseOptional(proxyBody).present()) {
+				throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
+				             proxyBody.c_str());
+			}
+			StringRef p(proxyBody);
 			proxyHost = p.eat(":").toString();
 			proxyPort = p.eat().toString();
 		}
@@ -645,10 +654,24 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 		}
 	}
 	std::string host = b->host, service = b->service;
-	if (service.empty())
+	if (service.empty()) {
+		if (b->useProxy) {
+			fprintf(stderr, "ERROR: Port can't be empty when using HTTP proxy.\n");
+			throw connection_failed();
+		}
 		service = b->knobs.secure_connection ? "https" : "http";
-	state Reference<IConnection> conn =
+	}
-	    wait(INetworkConnections::net()->connect(host, service, b->knobs.secure_connection ? true : false));
+	bool isTLS = b->knobs.secure_connection == 1;
+	if (b->useProxy) {
+		// TODO(renxuan): Support http proxy + TLS
+		if (isTLS || b->service == "443") {
+			fprintf(stderr, "ERROR: TLS is not supported yet when using HTTP proxy.\n");
+			throw connection_failed();
+		}
+		host = b->proxyHost.get();
+		service = b->proxyPort.get();
+	}
+	state Reference<IConnection> conn = wait(INetworkConnections::net()->connect(host, service, isTLS));
 	wait(conn->connectHandshake());
 
 	TraceEvent("S3BlobStoreEndpointNewConnection")
@@ -752,6 +775,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 				bstore->setAuthHeaders(verb, resource, headers);
 			}
 
+			if (bstore->useProxy) {
+				// Has to be in absolute-form.
+				resource = "http://" + bstore->host + ":" + bstore->service + resource;
+			}
 			remoteAddress = rconn.conn->getPeerAddress();
 			wait(bstore->requestRate->getAllowance(1));
 			Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn,

fdbclient/Schemas.cpp

@@ -927,6 +927,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
                "logical_core_utilization":0.4
             }
          }
+      },
+      "tenants":{
+         "num_tenants":0
       }
    },
    "client":{

fdbclient/ServerKnobs.cpp

@@ -673,6 +673,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( FETCH_KEYS_PARALLELISM,                                  2 );
 	init( FETCH_KEYS_LOWER_PRIORITY,                               0 );
 	init( FETCH_CHANGEFEED_PARALLELISM,                            2 );
+	init( SERVE_FETCH_CHECKPOINT_PARALLELISM,                      4 );
 	init( BUGGIFY_BLOCK_BYTES,                                 10000 );
 	init( STORAGE_RECOVERY_VERSION_LAG_LIMIT,				2 * MAX_READ_TRANSACTION_LIFE_VERSIONS );
 	init( STORAGE_COMMIT_BYTES,                             10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000;
@@ -857,6 +858,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
     init( ENABLE_ENCRYPTION,                                   false );
     init( ENCRYPTION_MODE,                              "AES-256-CTR");
     init( SIM_KMS_MAX_KEYS,                                      4096);
+    init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH,                  100000);
 
     // KMS connector type
     init( KMS_CONNECTOR_TYPE,                      "RESTKmsConnector");

fdbclient/ServerKnobs.h

@@ -622,6 +622,7 @@ public:
 	int FETCH_KEYS_PARALLELISM;
 	int FETCH_KEYS_LOWER_PRIORITY;
 	int FETCH_CHANGEFEED_PARALLELISM;
+	int SERVE_FETCH_CHECKPOINT_PARALLELISM;
 	int BUGGIFY_BLOCK_BYTES;
 	int64_t STORAGE_RECOVERY_VERSION_LAG_LIMIT;
 	double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD;
@@ -824,6 +825,7 @@ public:
 	bool ENABLE_ENCRYPTION;
 	std::string ENCRYPTION_MODE;
 	int SIM_KMS_MAX_KEYS;
+	int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
 
 	// Key Management Service (KMS) Connector
 	std::string KMS_CONNECTOR_TYPE;

fdbclient/SimpleConfigTransaction.h

@@ -63,6 +63,7 @@ public:
 	                                         KeySelector end,
 	                                         Key mapper,
 	                                         GetRangeLimits limits,
+	                                         int matchIndex,
 	                                         Snapshot = Snapshot::False,
 	                                         Reverse = Reverse::False) override {
 		throw client_invalid_operation();

fdbclient/SpecialKeySpace.actor.cpp

@@ -1464,11 +1464,9 @@ Future<RangeResult> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw,
                                                KeyRangeRef kr,
                                                GetRangeLimits limitsHint) const {
 	RangeResult result;
-
-	auto& globalConfig = GlobalConfig::globalConfig();
 	KeyRangeRef modified =
 	    KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin));
-	std::map<KeyRef, Reference<ConfigValue>> values = globalConfig.get(modified);
+	std::map<KeyRef, Reference<ConfigValue>> values = ryw->getDatabase()->globalConfig->get(modified);
 	for (const auto& [key, config] : values) {
 		Key prefixedKey = key.withPrefix(getKeyRange().begin);
 		if (config.isValid() && config->value.has_value()) {
@@ -1519,7 +1517,8 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
 		}
 	}
 
-	VersionHistory vh{ 0 };
+	Standalone<VectorRef<KeyValueRef>> insertions;
+	Standalone<VectorRef<KeyRangeRef>> clears;
 
 	// Transform writes from the special-key-space (\xff\xff/global_config/) to
 	// the system key space (\xff/globalConfig/), and writes mutations to
@@ -1532,36 +1531,17 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
 		if (entry.first) {
 			if (entry.second.present() && iter->begin().startsWith(globalConfig->getKeyRange().begin)) {
 				Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin);
-				vh.mutations.emplace_back_deep(vh.mutations.arena(),
+				insertions.push_back_deep(insertions.arena(), KeyValueRef(bareKey, entry.second.get()));
-				                               MutationRef(MutationRef::SetValue, bareKey, entry.second.get()));
-
-				Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix);
-				tr.set(systemKey, entry.second.get());
 			} else if (!entry.second.present() && iter->range().begin.startsWith(globalConfig->getKeyRange().begin) &&
 			           iter->range().end.startsWith(globalConfig->getKeyRange().begin)) {
 				KeyRef bareRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin);
 				KeyRef bareRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin);
-				vh.mutations.emplace_back_deep(vh.mutations.arena(),
+				clears.push_back_deep(clears.arena(), KeyRangeRef(bareRangeBegin, bareRangeEnd));
-				                               MutationRef(MutationRef::ClearRange, bareRangeBegin, bareRangeEnd));
-
-				Key systemRangeBegin = bareRangeBegin.withPrefix(globalConfigKeysPrefix);
-				Key systemRangeEnd = bareRangeEnd.withPrefix(globalConfigKeysPrefix);
-				tr.clear(KeyRangeRef(systemRangeBegin, systemRangeEnd));
 			}
 		}
 		++iter;
 	}
-
+	GlobalConfig::applyChanges(tr, insertions, clears);
-	// Record the mutations in this commit into the global configuration history.
-	Key historyKey = addVersionStampAtEnd(globalConfigHistoryPrefix);
-	ObjectWriter historyWriter(IncludeVersion());
-	historyWriter.serialize(vh);
-	tr.atomicOp(historyKey, historyWriter.toStringRef(), MutationRef::SetVersionstampedKey);
-
-	// Write version key to trigger update in cluster controller.
-	tr.atomicOp(globalConfigVersionKey,
-	            LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp
-	            MutationRef::SetVersionstampedValue);
 
 	return Optional<std::string>();
 }
@@ -1970,13 +1950,11 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 			ASSERT(entry.second.present());
 			result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, entry.second.get()));
 		} else {
-			Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSampleRate));
 			std::string sampleRateStr = "default";
-			if (f.present()) {
+			const double sampleRateDbl = ryw->getDatabase()->globalConfig->get<double>(
-				const double sampleRateDbl = BinaryReader::fromStringRef<double>(f.get(), Unversioned());
+			    fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
-				if (!std::isinf(sampleRateDbl)) {
+			if (!std::isinf(sampleRateDbl)) {
-					sampleRateStr = boost::lexical_cast<std::string>(sampleRateDbl);
+				sampleRateStr = std::to_string(sampleRateDbl);
-				}
 			}
 			result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, Value(sampleRateStr)));
 		}
@@ -1990,13 +1968,10 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 			ASSERT(entry.second.present());
 			result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, entry.second.get()));
 		} else {
-			Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSizeLimit));
 			std::string sizeLimitStr = "default";
-			if (f.present()) {
+			const int64_t sizeLimit = ryw->getDatabase()->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
-				const int64_t sizeLimit = BinaryReader::fromStringRef<int64_t>(f.get(), Unversioned());
+			if (sizeLimit != -1) {
-				if (sizeLimit != -1) {
+				sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
-					sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
-				}
 			}
 			result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, Value(sizeLimitStr)));
 		}
@@ -2013,43 +1988,49 @@ Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw
 Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransaction* ryw) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 
+	Standalone<VectorRef<KeyValueRef>> insertions;
+	Standalone<VectorRef<KeyRangeRef>> clears;
+
 	// client_txn_sample_rate
 	Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(getKeyRange().begin);
 	auto rateEntry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey];
 
 	if (rateEntry.first && rateEntry.second.present()) {
 		std::string sampleRateStr = rateEntry.second.get().toString();
-		double sampleRate;
+		if (sampleRateStr == "default") {
-		if (sampleRateStr == "default")
+			clears.push_back_deep(clears.arena(),
-			sampleRate = std::numeric_limits<double>::infinity();
+			                      KeyRangeRef(fdbClientInfoTxnSampleRate, keyAfter(fdbClientInfoTxnSampleRate)));
-		else {
+		} else {
 			try {
-				sampleRate = boost::lexical_cast<double>(sampleRateStr);
+				double sampleRate = boost::lexical_cast<double>(sampleRateStr);
+				Tuple rate = Tuple().appendDouble(sampleRate);
+				insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSampleRate, rate.pack()));
 			} catch (boost::bad_lexical_cast& e) {
 				return Optional<std::string>(ManagementAPIError::toJsonString(
 				    false, "profile", "Invalid transaction sample rate(double): " + sampleRateStr));
 			}
 		}
-		ryw->getTransaction().set(fdbClientInfoTxnSampleRate, BinaryWriter::toValue(sampleRate, Unversioned()));
 	}
 	// client_txn_size_limit
 	Key txnSizeLimitKey = LiteralStringRef("client_txn_size_limit").withPrefix(getKeyRange().begin);
 	auto sizeLimitEntry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey];
 	if (sizeLimitEntry.first && sizeLimitEntry.second.present()) {
 		std::string sizeLimitStr = sizeLimitEntry.second.get().toString();
-		int64_t sizeLimit;
+		if (sizeLimitStr == "default") {
-		if (sizeLimitStr == "default")
+			clears.push_back_deep(clears.arena(),
-			sizeLimit = -1;
+			                      KeyRangeRef(fdbClientInfoTxnSizeLimit, keyAfter(fdbClientInfoTxnSizeLimit)));
-		else {
+		} else {
 			try {
-				sizeLimit = boost::lexical_cast<int64_t>(sizeLimitStr);
+				int64_t sizeLimit = boost::lexical_cast<int64_t>(sizeLimitStr);
+				Tuple size = Tuple().append(sizeLimit);
+				insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSizeLimit, size.pack()));
 			} catch (boost::bad_lexical_cast& e) {
 				return Optional<std::string>(ManagementAPIError::toJsonString(
 				    false, "profile", "Invalid transaction size limit(int64_t): " + sizeLimitStr));
 			}
 		}
-		ryw->getTransaction().set(fdbClientInfoTxnSizeLimit, BinaryWriter::toValue(sizeLimit, Unversioned()));
 	}
+	GlobalConfig::applyChanges(ryw->getTransaction(), insertions, clears);
 	return Optional<std::string>();
 }
 

fdbclient/StorageServerInterface.h

@@ -426,6 +426,7 @@ struct GetMappedKeyValuesRequest : TimedRequest {
 	KeyRef mapper;
 	Version version; // or latestVersion
 	int limit, limitBytes;
+	int matchIndex;
 	bool isFetchKeys;
 	Optional<TagSet> tags;
 	Optional<UID> debugID;
@@ -451,7 +452,8 @@ struct GetMappedKeyValuesRequest : TimedRequest {
 		           spanContext,
 		           tenantInfo,
 		           arena,
-		           ssLatestCommitVersions);
+		           ssLatestCommitVersions,
+		           matchIndex);
 	}
 };
 

fdbclient/SystemData.cpp

@@ -302,7 +302,8 @@ std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID
 	return std::make_pair(logs, oldLogs);
 }
 
-const KeyRef serverKeysPrefix = "\xff/serverKeys/"_sr;
+const KeyRangeRef serverKeysRange = KeyRangeRef("\xff/serverKeys/"_sr, "\xff/serverKeys0"_sr);
+const KeyRef serverKeysPrefix = serverKeysRange.begin;
 const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTrue
     serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty.
     serverKeysFalse;
@@ -328,6 +329,18 @@ UID serverKeysDecodeServer(const KeyRef& key) {
 	rd >> server_id;
 	return server_id;
 }
+
+std::pair<UID, Key> serverKeysDecodeServerBegin(const KeyRef& key) {
+	UID server_id;
+	BinaryReader rd(key.removePrefix(serverKeysPrefix), Unversioned());
+	rd >> server_id;
+	rd.readBytes(1); // skip "/"
+	const auto remainingBytes = rd.remainingBytes();
+	KeyRef ref = KeyRef(rd.arenaRead(remainingBytes), remainingBytes);
+	// std::cout << ref.size() << " " << ref.toString() << std::endl;
+	return std::make_pair(server_id, Key(ref));
+}
+
 bool serverHasKey(ValueRef storedValue) {
 	return storedValue == serverKeysTrue || storedValue == serverKeysTrueEmptyRange;
 }

fdbclient/SystemData.h

@@ -99,11 +99,13 @@ void decodeStorageCacheValue(const ValueRef& value, std::vector<uint16_t>& serve
 //	Using the serverID as a prefix, then followed by the beginning of the shard range
 //	as the key, the value indicates whether the shard does or does not exist on the server.
 //	These values can be changed as data movement occurs.
+extern const KeyRangeRef serverKeysRange;
 extern const KeyRef serverKeysPrefix;
 extern const ValueRef serverKeysTrue, serverKeysTrueEmptyRange, serverKeysFalse;
 const Key serverKeysKey(UID serverID, const KeyRef& keys);
 const Key serverKeysPrefixFor(UID serverID);
 UID serverKeysDecodeServer(const KeyRef& key);
+std::pair<UID, Key> serverKeysDecodeServerBegin(const KeyRef& key);
 bool serverHasKey(ValueRef storedValue);
 
 extern const KeyRangeRef conflictingKeysRange;

fdbclient/Tenant.h

@@ -55,7 +55,9 @@ private:
 		ASSERT(id >= 0);
 		prefix = makeString(8 + subspace.size());
 		uint8_t* data = mutateString(prefix);
-		memcpy(data, subspace.begin(), subspace.size());
+		if (subspace.size() > 0) {
+			memcpy(data, subspace.begin(), subspace.size());
+		}
 		int64_t swapped = bigEndian64(id);
 		memcpy(data + subspace.size(), &swapped, 8);
 	}

fdbclient/ThreadSafeTransaction.cpp

@@ -306,6 +306,7 @@ ThreadFuture<MappedRangeResult> ThreadSafeTransaction::getMappedRange(const KeyS
                                                                       const KeySelectorRef& end,
                                                                       const StringRef& mapper,
                                                                       GetRangeLimits limits,
+                                                                      int matchIndex,
                                                                       bool snapshot,
                                                                       bool reverse) {
 	KeySelector b = begin;
@@ -313,9 +314,9 @@ ThreadFuture<MappedRangeResult> ThreadSafeTransaction::getMappedRange(const KeyS
 	Key h = mapper;
 
 	ISingleThreadTransaction* tr = this->tr;
-	return onMainThread([tr, b, e, h, limits, snapshot, reverse]() -> Future<MappedRangeResult> {
+	return onMainThread([tr, b, e, h, limits, matchIndex, snapshot, reverse]() -> Future<MappedRangeResult> {
 		tr->checkDeferredError();
-		return tr->getMappedRange(b, e, h, limits, Snapshot{ snapshot }, Reverse{ reverse });
+		return tr->getMappedRange(b, e, h, limits, matchIndex, Snapshot{ snapshot }, Reverse{ reverse });
 	});
 }
 

fdbclient/ThreadSafeTransaction.h

@@ -136,6 +136,7 @@ public:
 	                                               const KeySelectorRef& end,
 	                                               const StringRef& mapper,
 	                                               GetRangeLimits limits,
+	                                               int matchIndex,
 	                                               bool snapshot,
 	                                               bool reverse) override;
 	ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;

fdbclient/vexillographer/fdb.options

@@ -134,7 +134,7 @@ description is not currently required but encouraged.
     <Option name="distributed_client_tracer" code="90"
             paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, or network_lossy"
             description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
-    <Option name="client_tmp_dir" code="90"
+    <Option name="client_tmp_dir" code="91"
             paramType="String" paramDescription="Client directory for temporary files. "
             description="Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp" />
     <Option name="supported_client_versions" code="1000"

fdbrpc/FlowTransport.actor.cpp

@@ -479,6 +479,7 @@ struct ConnectPacket {
 		serializer(ar, connectPacketLength);
 		if (connectPacketLength > sizeof(ConnectPacket) - sizeof(connectPacketLength)) {
 			ASSERT(!g_network->isSimulated());
+			TraceEvent("SerializationFailed").backtrace();
 			throw serialization_failed();
 		}
 
@@ -816,6 +817,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 				    .errorUnsuppressed(e)
 				    .suppressFor(1.0)
 				    .detail("PeerAddr", self->destination);
+
+				// Since the connection has closed, we need to check the protocol version the next time we connect
+				self->compatible = true;
 			}
 
 			if (self->destination.isPublic() &&
@@ -885,9 +889,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 Peer::Peer(TransportData* transport, NetworkAddress const& destination)
   : transport(transport), destination(destination), compatible(true), outgoingConnectionIdle(true),
     lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), peerReferences(-1),
-    incompatibleProtocolVersionNewer(false), bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()),
+    bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()), outstandingReplies(0),
-    outstandingReplies(0), pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1),
+    pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1), lastLoggedTime(0.0),
-    lastLoggedTime(0.0), lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
+    lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
     protocolVersion(Reference<AsyncVar<Optional<ProtocolVersion>>>(new AsyncVar<Optional<ProtocolVersion>>())),
     connectOutgoingCount(0), connectIncomingCount(0), connectFailedCount(0),
     connectLatencies(destination.isPublic() ? FLOW_KNOBS->NETWORK_CONNECT_SAMPLE_AMOUNT : 1) {
@@ -1257,7 +1261,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 	state bool expectConnectPacket = true;
 	state bool compatible = false;
 	state bool incompatiblePeerCounted = false;
-	state bool incompatibleProtocolVersionNewer = false;
 	state NetworkAddress peerAddress;
 	state ProtocolVersion peerProtocolVersion;
 	state Reference<AuthorizedTenants> authorizedTenants = makeReference<AuthorizedTenants>();
@@ -1323,7 +1326,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 						uint64_t connectionId = pkt.connectionId;
 						if (!pkt.protocolVersion.hasObjectSerializerFlag() ||
 						    !pkt.protocolVersion.isCompatible(g_network->protocolVersion())) {
-							incompatibleProtocolVersionNewer = pkt.protocolVersion > g_network->protocolVersion();
 							NetworkAddress addr = pkt.canonicalRemotePort
 							                          ? NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort)
 							                          : conn->getPeerAddress();
@@ -1383,7 +1385,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 							    .suppressFor(1.0)
 							    .detail("PeerAddr", NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort));
 							peer->compatible = compatible;
-							peer->incompatibleProtocolVersionNewer = incompatibleProtocolVersionNewer;
 							if (!compatible) {
 								peer->transport->numIncompatibleConnections++;
 								incompatiblePeerCounted = true;
@@ -1401,7 +1402,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 							}
 							peer = transport->getOrOpenPeer(peerAddress, false);
 							peer->compatible = compatible;
-							peer->incompatibleProtocolVersionNewer = incompatibleProtocolVersionNewer;
 							if (!compatible) {
 								peer->transport->numIncompatibleConnections++;
 								incompatiblePeerCounted = true;
@@ -1741,8 +1741,7 @@ static ReliablePacket* sendPacket(TransportData* self,
 
 	// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
 	if (!peer || (peer->outgoingConnectionIdle && !destination.getPrimaryAddress().isPublic()) ||
-	    (peer->incompatibleProtocolVersionNewer &&
+	    (!peer->compatible && destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
-	     destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
 		TEST(true); // Can't send to private address without a compatible open connection
 		return nullptr;
 	}
@@ -1912,8 +1911,14 @@ Reference<AsyncVar<bool>> FlowTransport::getDegraded() {
 //
 // Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
 // version, some other mechanism should be used to connect to that peer.
-Reference<AsyncVar<Optional<ProtocolVersion>> const> FlowTransport::getPeerProtocolAsyncVar(NetworkAddress addr) {
+Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> FlowTransport::getPeerProtocolAsyncVar(
-	return self->peers.at(addr)->protocolVersion;
+    NetworkAddress addr) {
+	auto itr = self->peers.find(addr);
+	if (itr != self->peers.end()) {
+		return itr->second->protocolVersion;
+	} else {
+		return Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>>();
+	}
 }
 
 void FlowTransport::resetConnection(NetworkAddress address) {

fdbrpc/FlowTransport.h

@@ -159,7 +159,6 @@ struct Peer : public ReferenceCounted<Peer> {
 	double lastConnectTime;
 	double reconnectionDelay;
 	int peerReferences;
-	bool incompatibleProtocolVersionNewer;
 	int64_t bytesReceived;
 	int64_t bytesSent;
 	double lastDataPacketSentTime;
@@ -281,7 +280,7 @@ public:
 	//
 	// Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
 	// version, some other mechanism should be used to connect to that peer.
-	Reference<AsyncVar<Optional<ProtocolVersion>> const> getPeerProtocolAsyncVar(NetworkAddress addr);
+	Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> getPeerProtocolAsyncVar(NetworkAddress addr);
 
 	static FlowTransport& transport() {
 		return *static_cast<FlowTransport*>((void*)g_network->global(INetwork::enFlowTransport));

fdbrpc/TokenSign.cpp

@@ -24,10 +24,15 @@
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
+#include "flow/MkCert.h"
 #include "flow/Platform.h"
+#include "flow/ScopeExit.h"
 #include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include <type_traits>
+#if defined(HAVE_WOLFSSL)
+#include <wolfssl/options.h>
+#endif
 #include <openssl/ec.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
@@ -35,16 +40,6 @@
 
 namespace {
 
-template <typename Func>
-class ExitGuard {
-	std::decay_t<Func> fn;
-
-public:
-	ExitGuard(Func&& fn) : fn(std::forward<Func>(fn)) {}
-
-	~ExitGuard() { fn(); }
-};
-
 [[noreturn]] void traceAndThrow(const char* type) {
 	auto te = TraceEvent(SevWarnAlways, type);
 	te.suppressFor(60);
@@ -53,63 +48,11 @@ public:
 			0,
 		};
 		::ERR_error_string_n(err, buf, sizeof(buf));
-		te.detail("OpenSSLError", buf);
+		te.detail("OpenSSLError", static_cast<const char*>(buf));
 	}
 	throw digital_signature_ops_error();
 }
 
-struct KeyPairRef {
-	StringRef privateKey;
-	StringRef publicKey;
-};
-
-Standalone<KeyPairRef> generateEcdsaKeyPair() {
-	auto params = std::add_pointer_t<EVP_PKEY>();
-	{
-		auto pctx = ::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr);
-		ASSERT(pctx);
-		auto ctxGuard = ExitGuard([pctx]() { ::EVP_PKEY_CTX_free(pctx); });
-		ASSERT_LT(0, ::EVP_PKEY_paramgen_init(pctx));
-		ASSERT_LT(0, ::EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, NID_X9_62_prime256v1));
-		ASSERT_LT(0, ::EVP_PKEY_paramgen(pctx, &params));
-		ASSERT(params);
-	}
-	auto paramsGuard = ExitGuard([params]() { ::EVP_PKEY_free(params); });
-	// keygen
-	auto kctx = ::EVP_PKEY_CTX_new(params, nullptr);
-	ASSERT(kctx);
-	auto kctxGuard = ExitGuard([kctx]() { ::EVP_PKEY_CTX_free(kctx); });
-	auto key = std::add_pointer_t<EVP_PKEY>();
-	{
-		ASSERT_LT(0, ::EVP_PKEY_keygen_init(kctx));
-		ASSERT_LT(0, ::EVP_PKEY_keygen(kctx, &key));
-	}
-	ASSERT(key);
-	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
-
-	auto ret = Standalone<KeyPairRef>{};
-	auto& arena = ret.arena();
-	{
-		auto len = 0;
-		len = ::i2d_PrivateKey(key, nullptr);
-		ASSERT_LT(0, len);
-		auto buf = new (arena) uint8_t[len];
-		auto out = std::add_pointer_t<uint8_t>(buf);
-		len = ::i2d_PrivateKey(key, &out);
-		ret.privateKey = StringRef(buf, len);
-	}
-	{
-		auto len = 0;
-		len = ::i2d_PUBKEY(key, nullptr);
-		ASSERT_LT(0, len);
-		auto buf = new (arena) uint8_t[len];
-		auto out = std::add_pointer_t<uint8_t>(buf);
-		len = ::i2d_PUBKEY(key, &out);
-		ret.publicKey = StringRef(buf, len);
-	}
-	return ret;
-}
-
 } // namespace
 
 Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName, StringRef privateKeyDer) {
@@ -124,11 +67,11 @@ Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName,
 	if (!key) {
 		traceAndThrow("SignTokenBadKey");
 	}
-	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
+	auto keyGuard = ScopeExit([key]() { ::EVP_PKEY_free(key); });
 	auto mdctx = ::EVP_MD_CTX_create();
 	if (!mdctx)
 		traceAndThrow("SignTokenInitFail");
-	auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
+	auto mdctxGuard = ScopeExit([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
 	if (1 != ::EVP_DigestSignInit(mdctx, nullptr, ::EVP_sha256() /*Parameterize?*/, nullptr, key))
 		traceAndThrow("SignTokenInitFail");
 	if (1 != ::EVP_DigestSignUpdate(mdctx, tokenStr.begin(), tokenStr.size()))
@@ -150,11 +93,11 @@ bool verifyToken(SignedAuthTokenRef signedToken, StringRef publicKeyDer) {
 	auto key = ::d2i_PUBKEY(nullptr, &rawPubKeyDer, publicKeyDer.size());
 	if (!key)
 		traceAndThrow("VerifyTokenBadKey");
-	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
+	auto keyGuard = ScopeExit([key]() { ::EVP_PKEY_free(key); });
 	auto mdctx = ::EVP_MD_CTX_create();
 	if (!mdctx)
 		traceAndThrow("VerifyTokenInitFail");
-	auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
+	auto mdctxGuard = ScopeExit([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
 	if (1 != ::EVP_DigestVerifyInit(mdctx, nullptr, ::EVP_sha256(), nullptr, key))
 		traceAndThrow("VerifyTokenInitFail");
 	if (1 != ::EVP_DigestVerifyUpdate(mdctx, signedToken.token.begin(), signedToken.token.size()))
@@ -179,7 +122,8 @@ void forceLinkTokenSignTests() {}
 TEST_CASE("/fdbrpc/TokenSign") {
 	const auto numIters = 100;
 	for (auto i = 0; i < numIters; i++) {
-		auto keyPair = generateEcdsaKeyPair();
+		auto kpArena = Arena();
+		auto keyPair = mkcert::KeyPairRef::make(kpArena);
 		auto token = Standalone<AuthTokenRef>{};
 		auto& arena = token.arena();
 		auto& rng = *deterministicRandom();
@@ -206,15 +150,15 @@ TEST_CASE("/fdbrpc/TokenSign") {
 			token.tenants.push_back(arena, genRandomStringRef());
 		}
 		auto keyName = genRandomStringRef();
-		auto signedToken = signToken(token, keyName, keyPair.privateKey);
+		auto signedToken = signToken(token, keyName, keyPair.privateKeyDer);
-		const auto verifyExpectOk = verifyToken(signedToken, keyPair.publicKey);
+		const auto verifyExpectOk = verifyToken(signedToken, keyPair.publicKeyDer);
 		ASSERT(verifyExpectOk);
 		// try tampering with signed token by adding one more tenant
 		token.tenants.push_back(arena, genRandomStringRef());
 		auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
 		writer.serialize(token);
 		signedToken.token = writer.toStringRef();
-		const auto verifyExpectFail = verifyToken(signedToken, keyPair.publicKey);
+		const auto verifyExpectFail = verifyToken(signedToken, keyPair.publicKeyDer);
 		ASSERT(!verifyExpectFail);
 	}
 	printf("%d runs OK\n", numIters);

fdbrpc/genericactors.actor.h

@@ -101,7 +101,6 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request, Hostname h
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
 			// Connection failure.
-			hostname.resetToUnresolved();
 			INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 		}
 	}
@@ -126,7 +125,6 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
 			// Connection failure.
-			hostname.resetToUnresolved();
 			INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 		}
 	}
@@ -149,7 +147,6 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request, Hostname hostname
 				// Connection failure.
 				wait(delay(reconnetInterval));
 				reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
-				hostname.resetToUnresolved();
 				INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 			} else {
 				throw reply.getError();
@@ -179,7 +176,6 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
 				// Connection failure.
 				wait(delay(reconnetInterval));
 				reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
-				hostname.resetToUnresolved();
 				INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 			} else {
 				throw reply.getError();

fdbrpc/sim2.actor.cpp

@@ -106,14 +106,14 @@ void ISimulator::displayWorkers() const {
 		for (auto& processInfo : machineRecord.second) {
 			printf("                  %9s %-10s%-13s%-8s %-6s %-9s %-8s %-48s %-40s\n",
 			       processInfo->address.toString().c_str(),
-			       processInfo->name,
+			       processInfo->name.c_str(),
 			       processInfo->startingClass.toString().c_str(),
 			       (processInfo->isExcluded() ? "True" : "False"),
 			       (processInfo->failed ? "True" : "False"),
 			       (processInfo->rebooting ? "True" : "False"),
 			       (processInfo->isCleared() ? "True" : "False"),
 			       getRoles(processInfo->address).c_str(),
-			       processInfo->dataFolder);
+			       processInfo->dataFolder.c_str());
 		}
 	}
 
@@ -1588,7 +1588,7 @@ public:
 			    .detail("Protected", protectedAddresses.count(machine->address))
 			    .backtrace();
 			// This will remove all the "tracked" messages that came from the machine being killed
-			if (std::string(machine->name) != "remote flow process")
+			if (machine->name != "remote flow process")
 				latestEventCache.clear();
 			machine->failed = true;
 		} else if (kt == InjectFaults) {
@@ -1618,7 +1618,7 @@ public:
 			ASSERT(false);
 		}
 		ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting ||
-		       std::string(machine->name) == "remote flow process");
+		       machine->name == "remote flow process");
 	}
 	void rebootProcess(ProcessInfo* process, KillType kt) override {
 		if (kt == RebootProcessAndDelete && protectedAddresses.count(process->address)) {
@@ -2465,7 +2465,7 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 			    .detail("Rebooting", p->rebooting)
 			    .detail("Reliable", p->isReliable());
 			return;
-		} else if (std::string(p->name) == "remote flow process") {
+		} else if (p->name == "remote flow process") {
 			TraceEvent(SevDebug, "DoRebootFailed").detail("Name", p->name).detail("Address", p->address);
 			return;
 		} else if (p->getChilds().size()) {

fdbrpc/simulator.h

@@ -59,9 +59,9 @@ public:
 	struct MachineInfo;
 
 	struct ProcessInfo : NonCopyable {
-		const char* name;
+		std::string name;
-		const char* coordinationFolder;
+		std::string coordinationFolder;
-		const char* dataFolder;
+		std::string dataFolder;
 		MachineInfo* machine;
 		NetworkAddressList addresses;
 		NetworkAddress address;
@@ -182,7 +182,7 @@ public:
 		std::string toString() const {
 			return format(
 			    "name: %s address: %s zone: %s datahall: %s class: %s excluded: %d cleared: %d",
-			    name,
+			    name.c_str(),
 			    formatIpPort(addresses.address.ip, addresses.address.port).c_str(),
 			    (locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"),
 			    (locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"),

fdbserver/CMakeLists.txt

@@ -105,7 +105,7 @@ set(FDBSERVER_SRCS
   RecoveryState.h
   RemoteIKeyValueStore.actor.h
   RemoteIKeyValueStore.actor.cpp
-  RESTKmsConnector.actor.h
+  RESTKmsConnector.h
   RESTKmsConnector.actor.cpp
   ResolutionBalancer.actor.cpp
   ResolutionBalancer.actor.h
@@ -138,7 +138,7 @@ set(FDBSERVER_SRCS
   ServerDBInfo.actor.h
   ServerDBInfo.h
   SigStack.cpp
-  SimKmsConnector.actor.h
+  SimKmsConnector.h
   SimKmsConnector.actor.cpp
   SimpleConfigConsumer.actor.cpp
   SimpleConfigConsumer.h
@@ -149,7 +149,7 @@ set(FDBSERVER_SRCS
   Status.actor.cpp
   Status.h
   StorageCache.actor.cpp
-  StorageMetrics.actor.h
+  StorageMetrics.actor.cpp
   StorageMetrics.h
   storageserver.actor.cpp
   TagPartitionedLogSystem.actor.cpp
@@ -267,6 +267,7 @@ set(FDBSERVER_SRCS
   workloads/ReadAfterWrite.actor.cpp
   workloads/ReadHotDetection.actor.cpp
   workloads/ReadWrite.actor.cpp
+  workloads/ReadWriteWorkload.actor.h
   workloads/RemoveServersSafely.actor.cpp
   workloads/ReportConflictingKeys.actor.cpp
   workloads/RestoreBackup.actor.cpp
@@ -281,6 +282,7 @@ set(FDBSERVER_SRCS
   workloads/Sideband.actor.cpp
   workloads/SidebandSingle.actor.cpp
   workloads/SimpleAtomicAdd.actor.cpp
+  workloads/SkewedReadWrite.actor.cpp
   workloads/SlowTaskWorkload.actor.cpp
   workloads/SnapTest.actor.cpp
   workloads/SpecialKeySpaceCorrectness.actor.cpp

fdbserver/ClusterController.actor.cpp

@@ -3170,4 +3170,4 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer
 	return Void();
 }
 
-} // namespace
+} // namespace

fdbserver/ClusterRecovery.actor.cpp

@@ -1467,7 +1467,6 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	DBCoreState newState = self->cstate.myDBState;
 	newState.recoveryCount++;
-	newState.recoveryCount++;
 	if (self->cstate.prevDBState.newestProtocolVersion.isInvalid() ||
 	    self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) {
 		ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalid() ||

fdbserver/DDTeamCollection.actor.cpp

@@ -468,20 +468,20 @@ public:
 		self->healthyZone.set(initTeams->initHealthyZoneValue);
 		// SOMEDAY: If some servers have teams and not others (or some servers have more data than others) and there is
 		// an address/locality collision, should we preferentially mark the least used server as undesirable?
-		for (auto& server : initTeams->allServers) {
+		for (auto& [server, procClass] : initTeams->allServers) {
-			if (self->shouldHandleServer(server.first)) {
+			if (self->shouldHandleServer(server)) {
-				if (!self->isValidLocality(self->configuration.storagePolicy, server.first.locality)) {
+				if (!self->isValidLocality(self->configuration.storagePolicy, server.locality)) {
 					TraceEvent(SevWarnAlways, "MissingLocality")
-					    .detail("Server", server.first.uniqueID)
+					    .detail("Server", server.uniqueID)
-					    .detail("Locality", server.first.locality.toString());
+					    .detail("Locality", server.locality.toString());
-					auto addr = server.first.stableAddress();
+					auto addr = server.stableAddress();
 					self->invalidLocalityAddr.insert(AddressExclusion(addr.ip, addr.port));
 					if (self->checkInvalidLocalities.isReady()) {
 						self->checkInvalidLocalities = checkAndRemoveInvalidLocalityAddr(self);
 						self->addActor.send(self->checkInvalidLocalities);
 					}
 				}
-				self->addServer(server.first, server.second, self->serverTrackerErrorOut, 0, *ddEnabledState);
+				self->addServer(server, procClass, self->serverTrackerErrorOut, 0, *ddEnabledState);
 			}
 		}
 
@@ -514,13 +514,14 @@ public:
 			}
 		}
 
-		for (auto i = self->server_info.begin(); i != self->server_info.end(); ++i) {
+		for (auto& [serverID, server] : self->server_info) {
-			if (!self->server_status.get(i->first).isUnhealthy()) {
+			if (!self->server_status.get(serverID).isUnhealthy()) {
 				++serverCount;
-				LocalityData const& serverLocation = i->second->getLastKnownInterface().locality;
+				LocalityData const& serverLocation = server->getLastKnownInterface().locality;
 				machines.insert(serverLocation.zoneId());
 			}
 		}
+
 		uniqueMachines = machines.size();
 		TraceEvent("BuildTeams", self->distributorId)
 		    .detail("ServerCount", self->server_info.size())
@@ -594,8 +595,8 @@ public:
 				int maxMachineTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * totalHealthyMachineCount;
 				int healthyMachineTeamCount = self->getHealthyMachineTeamCount();
 
-				std::pair<uint64_t, uint64_t> minMaxTeamsOnServer = self->calculateMinMaxServerTeamsOnServer();
+				auto [minTeamsOnServer, maxTeamsOnServer] = self->calculateMinMaxServerTeamsOnServer();
-				std::pair<uint64_t, uint64_t> minMaxMachineTeamsOnMachine =
+				auto [minMachineTeamsOnMachine, maxMachineTeamsOnMachine] =
 				    self->calculateMinMaxMachineTeamsOnMachine();
 
 				TraceEvent("TeamCollectionInfo", self->distributorId)
@@ -611,10 +612,10 @@ public:
 				    .detail("DesiredMachineTeams", desiredMachineTeams)
 				    .detail("MaxMachineTeams", maxMachineTeams)
 				    .detail("TotalHealthyMachines", totalHealthyMachineCount)
-				    .detail("MinTeamsOnServer", minMaxTeamsOnServer.first)
+				    .detail("MinTeamsOnServer", minTeamsOnServer)
-				    .detail("MaxTeamsOnServer", minMaxTeamsOnServer.second)
+				    .detail("MaxTeamsOnServer", maxTeamsOnServer)
-				    .detail("MinMachineTeamsOnMachine", minMaxMachineTeamsOnMachine.first)
+				    .detail("MinMachineTeamsOnMachine", maxMachineTeamsOnMachine)
-				    .detail("MaxMachineTeamsOnMachine", minMaxMachineTeamsOnMachine.second)
+				    .detail("MaxMachineTeamsOnMachine", minMachineTeamsOnMachine)
 				    .detail("DoBuildTeams", self->doBuildTeams)
 				    .trackLatest(self->teamCollectionInfoEventHolder->trackingKey);
 			}
@@ -3257,24 +3258,24 @@ void DDTeamCollection::traceServerInfo() const {
 	int i = 0;
 
 	TraceEvent("ServerInfo", distributorId).detail("Size", server_info.size());
-	for (auto& server : server_info) {
+	for (auto& [serverID, server] : server_info) {
 		TraceEvent("ServerInfo", distributorId)
 		    .detail("ServerInfoIndex", i++)
-		    .detail("ServerID", server.first.toString())
+		    .detail("ServerID", serverID.toString())
-		    .detail("ServerTeamOwned", server.second->getTeams().size())
+		    .detail("ServerTeamOwned", server->getTeams().size())
-		    .detail("MachineID", server.second->machine->machineID.contents().toString())
+		    .detail("MachineID", server->machine->machineID.contents().toString())
-		    .detail("StoreType", server.second->getStoreType().toString())
+		    .detail("StoreType", server->getStoreType().toString())
-		    .detail("InDesiredDC", server.second->isInDesiredDC());
+		    .detail("InDesiredDC", server->isInDesiredDC());
 	}
-	for (auto& server : server_info) {
+	for (auto& [serverID, server] : server_info) {
-		const UID& uid = server.first;
 		TraceEvent("ServerStatus", distributorId)
-		    .detail("ServerID", uid)
+		    .detail("ServerID", serverID)
-		    .detail("Healthy", !server_status.get(uid).isUnhealthy())
+		    .detail("Healthy", !server_status.get(serverID).isUnhealthy())
-		    .detail("MachineIsValid", get(server_info, uid)->machine.isValid())
+		    .detail("MachineIsValid", get(server_info, serverID)->machine.isValid())
 		    .detail("MachineTeamSize",
-		            get(server_info, uid)->machine.isValid() ? get(server_info, uid)->machine->machineTeams.size()
+		            get(server_info, serverID)->machine.isValid()
-		                                                     : -1);
+		                ? get(server_info, serverID)->machine->machineTeams.size()
+		                : -1);
 	}
 }
 
@@ -3838,16 +3839,16 @@ void DDTeamCollection::addTeam(const std::vector<Reference<TCServerInfo>>& newTe
 
 	// For a good team, we add it to teams and create machine team for it when necessary
 	teams.push_back(teamInfo);
-	for (int i = 0; i < newTeamServers.size(); ++i) {
+	for (auto& server : newTeamServers) {
-		newTeamServers[i]->addTeam(teamInfo);
+		server->addTeam(teamInfo);
 	}
 
 	// Find or create machine team for the server team
 	// Add the reference of machineTeam (with machineIDs) into process team
 	std::vector<Standalone<StringRef>> machineIDs;
-	for (auto server = newTeamServers.begin(); server != newTeamServers.end(); ++server) {
+	for (auto& server : newTeamServers) {
-		ASSERT_WE_THINK((*server)->machine.isValid());
+		ASSERT_WE_THINK(server->machine.isValid());
-		machineIDs.push_back((*server)->machine->machineID);
+		machineIDs.push_back(server->machine->machineID);
 	}
 	sort(machineIDs.begin(), machineIDs.end());
 	Reference<TCMachineTeamInfo> machineTeamInfo = findMachineTeam(machineIDs);
@@ -3907,9 +3908,9 @@ Reference<TCMachineTeamInfo> DDTeamCollection::addMachineTeam(std::vector<Standa
 
 int DDTeamCollection::constructMachinesFromServers() {
 	int totalServerIndex = 0;
-	for (auto i = server_info.begin(); i != server_info.end(); ++i) {
+	for (auto& [serverID, server] : server_info) {
-		if (!server_status.get(i->first).isUnhealthy()) {
+		if (!server_status.get(serverID).isUnhealthy()) {
-			checkAndCreateMachine(i->second);
+			checkAndCreateMachine(server);
 			totalServerIndex++;
 		}
 	}
@@ -4021,26 +4022,26 @@ void DDTeamCollection::traceAllInfo(bool shouldPrint) const {
 void DDTeamCollection::rebuildMachineLocalityMap() {
 	machineLocalityMap.clear();
 	int numHealthyMachine = 0;
-	for (auto machine = machine_info.begin(); machine != machine_info.end(); ++machine) {
+	for (auto& [_, machine] : machine_info) {
-		if (machine->second->serversOnMachine.empty()) {
+		if (machine->serversOnMachine.empty()) {
 			TraceEvent(SevWarn, "RebuildMachineLocalityMapError")
-			    .detail("Machine", machine->second->machineID.toString())
+			    .detail("Machine", machine->machineID.toString())
 			    .detail("NumServersOnMachine", 0);
 			continue;
 		}
-		if (!isMachineHealthy(machine->second)) {
+		if (!isMachineHealthy(machine)) {
 			continue;
 		}
-		Reference<TCServerInfo> representativeServer = machine->second->serversOnMachine[0];
+		Reference<TCServerInfo> representativeServer = machine->serversOnMachine[0];
 		auto& locality = representativeServer->getLastKnownInterface().locality;
 		if (!isValidLocality(configuration.storagePolicy, locality)) {
 			TraceEvent(SevWarn, "RebuildMachineLocalityMapError")
-			    .detail("Machine", machine->second->machineID.toString())
+			    .detail("Machine", machine->machineID.toString())
 			    .detail("InvalidLocality", locality.toString());
 			continue;
 		}
 		const LocalityEntry& localityEntry = machineLocalityMap.add(locality, &representativeServer->getId());
-		machine->second->localityEntry = localityEntry;
+		machine->localityEntry = localityEntry;
 		++numHealthyMachine;
 	}
 }
@@ -4420,12 +4421,12 @@ bool DDTeamCollection::notEnoughMachineTeamsForAMachine() const {
 	    SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS
 	        ? (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2
 	        : SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER;
-	for (auto& m : machine_info) {
+	for (auto& [_, machine] : machine_info) {
 		// If SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS is false,
 		// The desired machine team number is not the same with the desired server team number
 		// in notEnoughTeamsForAServer() below, because the machineTeamRemover() does not
 		// remove a machine team with the most number of machine teams.
-		if (m.second->machineTeams.size() < targetMachineTeamNumPerMachine && isMachineHealthy(m.second)) {
+		if (machine->machineTeams.size() < targetMachineTeamNumPerMachine && isMachineHealthy(machine)) {
 			return true;
 		}
 	}
@@ -4443,8 +4444,8 @@ bool DDTeamCollection::notEnoughTeamsForAServer() const {
 	// (#servers * DESIRED_TEAMS_PER_SERVER * storageTeamSize) / #servers.
 	int targetTeamNumPerServer = (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2;
 	ASSERT_GT(targetTeamNumPerServer, 0);
-	for (auto& s : server_info) {
+	for (auto& [serverID, server] : server_info) {
-		if (s.second->getTeams().size() < targetTeamNumPerServer && !server_status.get(s.first).isUnhealthy()) {
+		if (server->getTeams().size() < targetTeamNumPerServer && !server_status.get(serverID).isUnhealthy()) {
 			return true;
 		}
 	}
@@ -4651,11 +4652,11 @@ void DDTeamCollection::traceTeamCollectionInfo() const {
 void DDTeamCollection::noHealthyTeams() const {
 	std::set<UID> desiredServerSet;
 	std::string desc;
-	for (auto i = server_info.begin(); i != server_info.end(); ++i) {
+	for (auto& [serverID, server] : server_info) {
-		ASSERT(i->first == i->second->getId());
+		ASSERT(serverID == server->getId());
-		if (!server_status.get(i->first).isFailed) {
+		if (!server_status.get(serverID).isFailed) {
-			desiredServerSet.insert(i->first);
+			desiredServerSet.insert(serverID);
-			desc += i->first.shortString() + " (" + i->second->getLastKnownInterface().toString() + "), ";
+			desc += serverID.shortString() + " (" + server->getLastKnownInterface().toString() + "), ";
 		}
 	}
 

fdbserver/DataDistributionQueue.actor.cpp

@@ -355,8 +355,8 @@ bool canLaunchDest(const std::vector<std::pair<Reference<IDataDistributionTeam>,
 		return true;
 	}
 	int workFactor = getDestWorkFactor();
-	for (auto& team : candidateTeams) {
+	for (auto& [team, _] : candidateTeams) {
-		for (UID id : team.first->getServerIDs()) {
+		for (UID id : team->getServerIDs()) {
 			if (!busymapDest[id].canLaunch(priority, workFactor)) {
 				return false;
 			}
@@ -378,8 +378,8 @@ void launchDest(RelocateData& relocation,
                 std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.completeDests.empty());
 	int destWorkFactor = getDestWorkFactor();
-	for (auto& team : candidateTeams) {
+	for (auto& [team, _] : candidateTeams) {
-		for (UID id : team.first->getServerIDs()) {
+		for (UID id : team->getServerIDs()) {
 			relocation.completeDests.push_back(id);
 			destBusymap[id].addWork(relocation.priority, destWorkFactor);
 		}

fdbserver/EncryptKeyProxy.actor.cpp

@@ -24,9 +24,9 @@
 #include "fdbserver/KmsConnector.h"
 #include "fdbserver/KmsConnectorInterface.h"
 #include "fdbserver/Knobs.h"
-#include "fdbserver/RESTKmsConnector.actor.h"
+#include "fdbserver/RESTKmsConnector.h"
 #include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/SimKmsConnector.actor.h"
+#include "fdbserver/SimKmsConnector.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "flow/Arena.h"
@@ -42,6 +42,7 @@
 #include "flow/network.h"
 
 #include <boost/mpl/not.hpp>
+#include <string>
 #include <utility>
 #include <memory>
 
@@ -79,7 +80,12 @@ struct EncryptBaseCipherKey {
 };
 
 using EncryptBaseDomainIdCache = std::unordered_map<EncryptCipherDomainId, EncryptBaseCipherKey>;
-using EncryptBaseCipherKeyIdCache = std::unordered_map<EncryptCipherBaseKeyId, EncryptBaseCipherKey>;
+
+using EncryptBaseCipherDomainIdKeyIdCacheKey = std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>;
+using EncryptBaseCipherDomainIdKeyIdCacheKeyHash = boost::hash<EncryptBaseCipherDomainIdKeyIdCacheKey>;
+using EncryptBaseCipherDomainIdKeyIdCache = std::unordered_map<EncryptBaseCipherDomainIdKeyIdCacheKey,
+                                                               EncryptBaseCipherKey,
+                                                               EncryptBaseCipherDomainIdKeyIdCacheKeyHash>;
 
 struct EncryptKeyProxyData : NonCopyable, ReferenceCounted<EncryptKeyProxyData> {
 public:
@@ -88,7 +94,7 @@ public:
 	Future<Void> encryptionKeyRefresher;
 
 	EncryptBaseDomainIdCache baseCipherDomainIdCache;
-	EncryptBaseCipherKeyIdCache baseCipherKeyIdCache;
+	EncryptBaseCipherDomainIdKeyIdCache baseCipherDomainIdKeyIdCache;
 
 	std::unique_ptr<KmsConnector> kmsConnector;
 
@@ -112,6 +118,12 @@ public:
 	    numResponseWithErrors("EKPNumResponseWithErrors", ekpCacheMetrics),
 	    numEncryptionKeyRefreshErrors("EKPNumEncryptionKeyRefreshErrors", ekpCacheMetrics) {}
 
+	EncryptBaseCipherDomainIdKeyIdCacheKey getBaseCipherDomainIdKeyIdCacheKey(
+	    const EncryptCipherDomainId domainId,
+	    const EncryptCipherBaseKeyId baseCipherId) {
+		return std::make_pair(domainId, baseCipherId);
+	}
+
 	void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
 	                                 const EncryptCipherBaseKeyId baseCipherId,
 	                                 const StringRef baseCipherKey) {
@@ -130,7 +142,8 @@ public:
 		// Given an cipherKey is immutable, it is OK to NOT expire cached information.
 		// TODO: Update cache to support LRU eviction policy to limit the total cache size.
 
-		baseCipherKeyIdCache[baseCipherId] = EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, true);
+		EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId);
+		baseCipherDomainIdKeyIdCache[cacheKey] = EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, true);
 	}
 
 	template <class Reply>
@@ -162,10 +175,17 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 	// for the rest, reachout to KMS to fetch the required details
 
 	std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> lookupCipherIds;
-	state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
 
+	state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
 	state EKPGetBaseCipherKeysByIdsRequest keysByIds = req;
 	state EKPGetBaseCipherKeysByIdsReply keyIdsReply;
+	state Optional<TraceEvent> dbgTrace =
+	    keysByIds.debugId.present() ? TraceEvent("GetByKeyIds", ekpProxyData->myId) : Optional<TraceEvent>();
+
+	if (dbgTrace.present()) {
+		dbgTrace.get().setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
+		dbgTrace.get().detail("DbgId", keysByIds.debugId.get());
+	}
 
 	// Dedup the requested pair<baseCipherId, encryptDomainId>
 	// TODO: endpoint serialization of std::unordered_set isn't working at the moment
@@ -176,12 +196,30 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 		dedupedCipherIds.emplace(item);
 	}
 
+	if (dbgTrace.present()) {
+		dbgTrace.get().detail("NKeys", dedupedCipherIds.size());
+		for (const auto& item : dedupedCipherIds) {
+			// Record {encryptDomainId, baseCipherId} queried
+			dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.second, item.first), "");
+		}
+	}
+
 	for (const auto& item : dedupedCipherIds) {
-		const auto itr = ekpProxyData->baseCipherKeyIdCache.find(item.first);
+		const EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey =
-		if (itr != ekpProxyData->baseCipherKeyIdCache.end()) {
+		    ekpProxyData->getBaseCipherDomainIdKeyIdCacheKey(item.second, item.first);
+		const auto itr = ekpProxyData->baseCipherDomainIdKeyIdCache.find(cacheKey);
+		if (itr != ekpProxyData->baseCipherDomainIdKeyIdCache.end()) {
 			ASSERT(itr->second.isValid());
 			cachedCipherDetails.emplace_back(
 			    itr->second.domainId, itr->second.baseCipherId, itr->second.baseCipherKey, keyIdsReply.arena);
+
+			if (dbgTrace.present()) {
+				// {encryptId, baseCipherId} forms a unique tuple across encryption domains
+				dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
+				                                            itr->second.domainId,
+				                                            itr->second.baseCipherId),
+				                      "");
+			}
 		} else {
 			lookupCipherIds.emplace_back(std::make_pair(item.first, item.second));
 		}
@@ -192,7 +230,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 
 	if (!lookupCipherIds.empty()) {
 		try {
-			KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds);
+			KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds, keysByIds.debugId);
 			KmsConnLookupEKsByKeyIdsRep keysByIdsRep = wait(kmsConnectorInf.ekLookupByIds.getReply(keysByIdsReq));
 
 			for (const auto& item : keysByIdsRep.cipherKeyDetails) {
@@ -206,13 +244,20 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 			for (auto& item : keysByIdsRep.cipherKeyDetails) {
 				// DomainId isn't available here, the caller must know the encryption domainId
 				ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
+
+				if (dbgTrace.present()) {
+					// {encryptId, baseCipherId} forms a unique tuple across encryption domains
+					dbgTrace.get().detail(
+					    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
+					    "");
+				}
 			}
 		} catch (Error& e) {
 			if (!canReplyWith(e)) {
-				TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e);
+				TraceEvent("GetCipherKeysByKeyIds", ekpProxyData->myId).error(e);
 				throw;
 			}
-			TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code());
+			TraceEvent("GetCipherKeysByKeyIds", ekpProxyData->myId).detail("ErrorCode", e.code());
 			ekpProxyData->sendErrorResponse(keysByIds.reply, e);
 			return Void();
 		}
@@ -237,6 +282,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 	state EKPGetLatestBaseCipherKeysRequest latestKeysReq = req;
 	state EKPGetLatestBaseCipherKeysReply latestCipherReply;
 	state Arena& arena = latestCipherReply.arena;
+	state Optional<TraceEvent> dbgTrace =
+	    latestKeysReq.debugId.present() ? TraceEvent("GetByDomIds", ekpProxyData->myId) : Optional<TraceEvent>();
+
+	if (dbgTrace.present()) {
+		dbgTrace.get().setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
+		dbgTrace.get().detail("DbgId", latestKeysReq.debugId.get());
+	}
 
 	// Dedup the requested domainIds.
 	// TODO: endpoint serialization of std::unordered_set isn't working at the moment
@@ -245,6 +297,14 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 		dedupedDomainIds.emplace(id);
 	}
 
+	if (dbgTrace.present()) {
+		dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
+		for (EncryptCipherDomainId id : dedupedDomainIds) {
+			// log encryptDomainIds queried
+			dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, id), "");
+		}
+	}
+
 	// First, check if the requested information is already cached by the server.
 	// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
 
@@ -253,6 +313,12 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 		const auto itr = ekpProxyData->baseCipherDomainIdCache.find(id);
 		if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
 			cachedCipherDetails.emplace_back(id, itr->second.baseCipherId, itr->second.baseCipherKey, arena);
+
+			if (dbgTrace.present()) {
+				// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
+				dbgTrace.get().detail(
+				    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX, id, itr->second.baseCipherId), "");
+			}
 		} else {
 			lookupCipherDomains.emplace_back(id);
 		}
@@ -263,7 +329,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 
 	if (!lookupCipherDomains.empty()) {
 		try {
-			KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains);
+			KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains, latestKeysReq.debugId);
 			KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep =
 			    wait(kmsConnectorInf.ekLookupByDomainIds.getReply(keysByDomainIdReq));
 
@@ -273,6 +339,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 
 				// Record the fetched cipher details to the local cache for the future references
 				ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
+
+				if (dbgTrace.present()) {
+					// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
+					dbgTrace.get().detail(
+					    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
+					    "");
+				}
 			}
 		} catch (Error& e) {
 			if (!canReplyWith(e)) {
@@ -298,13 +371,16 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 
 ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpProxyData,
                                              KmsConnectorInterface kmsConnectorInf) {
+	state UID debugId = deterministicRandom()->randomUniqueID();
 
-	ASSERT(g_network->isSimulated());
+	state TraceEvent t("RefreshEKs_Start", ekpProxyData->myId);
-
+	t.setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
-	TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("KmsConnInf", kmsConnectorInf.id());
+	t.detail("KmsConnInf", kmsConnectorInf.id());
+	t.detail("DebugId", debugId);
 
 	try {
 		KmsConnLookupEKsByDomainIdsReq req;
+		req.debugId = debugId;
 		req.encryptDomainIds.reserve(ekpProxyData->baseCipherDomainIdCache.size());
 
 		for (auto& item : ekpProxyData->baseCipherDomainIdCache) {
@@ -313,16 +389,20 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
 		KmsConnLookupEKsByDomainIdsRep rep = wait(kmsConnectorInf.ekLookupByDomainIds.getReply(req));
 		for (auto& item : rep.cipherKeyDetails) {
 			ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
+			// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
+			t.detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
+			         "");
 		}
 
 		ekpProxyData->baseCipherKeysRefreshed += rep.cipherKeyDetails.size();
-		TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.cipherKeyDetails.size());
+
+		t.detail("nKeys", rep.cipherKeyDetails.size());
 	} catch (Error& e) {
 		if (!canReplyWith(e)) {
-			TraceEvent("RefreshEncryptionKeys_Error").error(e);
+			TraceEvent("RefreshEKs_Error").error(e);
 			throw e;
 		}
-		TraceEvent("RefreshEncryptionKeys").detail("ErrorCode", e.code());
+		TraceEvent("RefreshEKs").detail("ErrorCode", e.code());
 		++ekpProxyData->numEncryptionKeyRefreshErrors;
 	}
 

fdbserver/EncryptKeyProxyInterface.h

@@ -125,6 +125,7 @@ struct EKPGetBaseCipherKeysByIdsRequest {
 	constexpr static FileIdentifier file_identifier = 4930263;
 	UID requesterID;
 	std::vector<std::pair<uint64_t, int64_t>> baseCipherIds;
+	Optional<UID> debugId;
 	ReplyPromise<EKPGetBaseCipherKeysByIdsReply> reply;
 
 	EKPGetBaseCipherKeysByIdsRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
@@ -133,7 +134,7 @@ struct EKPGetBaseCipherKeysByIdsRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, requesterID, baseCipherIds, reply);
+		serializer(ar, requesterID, baseCipherIds, debugId, reply);
 	}
 };
 
@@ -156,17 +157,16 @@ struct EKPGetLatestBaseCipherKeysReply {
 
 struct EKPGetLatestBaseCipherKeysRequest {
 	constexpr static FileIdentifier file_identifier = 1910123;
-	UID requesterID;
 	std::vector<uint64_t> encryptDomainIds;
+	Optional<UID> debugId;
 	ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
 
-	EKPGetLatestBaseCipherKeysRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
+	EKPGetLatestBaseCipherKeysRequest() {}
-	explicit EKPGetLatestBaseCipherKeysRequest(UID uid, const std::vector<uint64_t>& ids)
+	explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<uint64_t>& ids) : encryptDomainIds(ids) {}
-	  : requesterID(uid), encryptDomainIds(ids) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, requesterID, encryptDomainIds, reply);
+		serializer(ar, encryptDomainIds, debugId, reply);
 	}
 };
 

fdbserver/FDBExecHelper.actor.cpp

@@ -160,16 +160,17 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
 		}
 	}
 	state int result = 0;
-	child = g_pSimulator->newProcess("remote flow process",
+	child = g_pSimulator->newProcess(
-	                                 self->address.ip,
+	    "remote flow process",
-	                                 0,
+	    self->address.ip,
-	                                 self->address.isTLS(),
+	    0,
-	                                 self->addresses.secondaryAddress.present() ? 2 : 1,
+	    self->address.isTLS(),
-	                                 self->locality,
+	    self->addresses.secondaryAddress.present() ? 2 : 1,
-	                                 ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource),
+	    self->locality,
-	                                 self->dataFolder,
+	    ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource),
-	                                 self->coordinationFolder, // do we need to customize this coordination folder path?
+	    self->dataFolder.c_str(),
-	                                 self->protocolVersion);
+	    self->coordinationFolder.c_str(), // do we need to customize this coordination folder path?
+	    self->protocolVersion);
 	wait(g_pSimulator->onProcess(child));
 	state Future<ISimulator::KillType> onShutdown = child->onShutdown();
 	state Future<ISimulator::KillType> parentShutdown = self->onShutdown();
@@ -268,7 +269,7 @@ static auto fork_child(const std::string& path, std::vector<char*>& paramList) {
 static void setupTraceWithOutput(TraceEvent& event, size_t bytesRead, char* outputBuffer) {
 	// get some errors printed for spawned process
 	std::cout << "Output bytesRead: " << bytesRead << std::endl;
-	std::cout << "output buffer: " << std::string(outputBuffer) << std::endl;
+	std::cout << "output buffer: " << std::string_view(outputBuffer, bytesRead) << std::endl;
 	if (bytesRead == 0)
 		return;
 	ASSERT(bytesRead <= SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT);

fdbserver/KeyValueStoreRocksDB.actor.cpp

@@ -846,8 +846,10 @@ ACTOR Future<Void> rocksDBMetricLogger(std::shared_ptr<rocksdb::Statistics> stat
 	}
 }
 
-void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
+void logRocksDBError(const rocksdb::Status& status,
-	auto level = status.IsTimedOut() ? SevWarn : SevError;
+                     const std::string& method,
+                     Optional<Severity> sev = Optional<Severity>()) {
+	Severity level = sev.present() ? sev.get() : (status.IsTimedOut() ? SevWarn : SevError);
 	TraceEvent e(level, "RocksDBError");
 	e.detail("Error", status.ToString()).detail("Method", method).detail("RocksDBSeverity", status.severity());
 	if (status.IsIOError()) {
@@ -867,9 +869,28 @@ Error statusToError(const rocksdb::Status& s) {
 
 struct RocksDBKeyValueStore : IKeyValueStore {
 	struct Writer : IThreadPoolReceiver {
+		struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
+			CheckpointAction(const CheckpointRequest& request) : request(request) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			const CheckpointRequest request;
+			ThreadReturnPromise<CheckpointMetaData> reply;
+		};
+
+		struct RestoreAction : TypedAction<Writer, RestoreAction> {
+			RestoreAction(const std::string& path, const std::vector<CheckpointMetaData>& checkpoints)
+			  : path(path), checkpoints(checkpoints) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			const std::string path;
+			const std::vector<CheckpointMetaData> checkpoints;
+			ThreadReturnPromise<Void> done;
+		};
+
 		DB& db;
 		CF& cf;
-
 		UID id;
 		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
 		std::shared_ptr<ReadIteratorPool> readIterPool;
@@ -1153,127 +1174,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			a.done.send(Void());
 		}
 
-		struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
+		void action(CheckpointAction& a);
-			CheckpointAction(const CheckpointRequest& request) : request(request) {}
 
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+		void action(RestoreAction& a);
-
-			const CheckpointRequest request;
-			ThreadReturnPromise<CheckpointMetaData> reply;
-		};
-
-		void action(CheckpointAction& a) {
-			TraceEvent("RocksDBServeCheckpointBegin", id)
-			    .detail("MinVersion", a.request.version)
-			    .detail("Range", a.request.range.toString())
-			    .detail("Format", static_cast<int>(a.request.format))
-			    .detail("CheckpointDir", a.request.checkpointDir);
-
-			rocksdb::Checkpoint* checkpoint;
-			rocksdb::Status s = rocksdb::Checkpoint::Create(db, &checkpoint);
-			if (!s.ok()) {
-				logRocksDBError(s, "Checkpoint");
-				a.reply.sendError(statusToError(s));
-				return;
-			}
-
-			rocksdb::PinnableSlice value;
-			rocksdb::ReadOptions readOptions = getReadOptions();
-			s = db->Get(readOptions, cf, toSlice(persistVersion), &value);
-
-			if (!s.ok() && !s.IsNotFound()) {
-				logRocksDBError(s, "Checkpoint");
-				a.reply.sendError(statusToError(s));
-				return;
-			}
-
-			const Version version = s.IsNotFound()
-			                            ? latestVersion
-			                            : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
-
-			TraceEvent("RocksDBServeCheckpointVersion", id)
-			    .detail("CheckpointVersion", a.request.version)
-			    .detail("PersistVersion", version);
-
-			// TODO: set the range as the actual shard range.
-			CheckpointMetaData res(version, a.request.range, a.request.format, a.request.checkpointID);
-			const std::string& checkpointDir = a.request.checkpointDir;
-
-			if (a.request.format == RocksDBColumnFamily) {
-				rocksdb::ExportImportFilesMetaData* pMetadata;
-				platform::eraseDirectoryRecursive(checkpointDir);
-				const std::string cwd = platform::getWorkingDirectory() + "/";
-				s = checkpoint->ExportColumnFamily(cf, checkpointDir, &pMetadata);
-
-				if (!s.ok()) {
-					logRocksDBError(s, "Checkpoint");
-					a.reply.sendError(statusToError(s));
-					return;
-				}
-
-				populateMetaData(&res, *pMetadata);
-				delete pMetadata;
-				TraceEvent("RocksDBServeCheckpointSuccess", id)
-				    .detail("CheckpointMetaData", res.toString())
-				    .detail("RocksDBCF", getRocksCF(res).toString());
-			} else {
-				throw not_implemented();
-			}
-
-			res.setState(CheckpointMetaData::Complete);
-			a.reply.send(res);
-		}
-
-		struct RestoreAction : TypedAction<Writer, RestoreAction> {
-			RestoreAction(const std::string& path, const std::vector<CheckpointMetaData>& checkpoints)
-			  : path(path), checkpoints(checkpoints) {}
-
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-
-			const std::string path;
-			const std::vector<CheckpointMetaData> checkpoints;
-			ThreadReturnPromise<Void> done;
-		};
-
-		void action(RestoreAction& a) {
-			TraceEvent("RocksDBServeRestoreBegin", id).detail("Path", a.path);
-
-			// TODO: Fail gracefully.
-			ASSERT(!a.checkpoints.empty());
-
-			if (a.checkpoints[0].format == RocksDBColumnFamily) {
-				ASSERT_EQ(a.checkpoints.size(), 1);
-				TraceEvent("RocksDBServeRestoreCF", id)
-				    .detail("Path", a.path)
-				    .detail("Checkpoint", a.checkpoints[0].toString())
-				    .detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
-
-				auto options = getOptions();
-				rocksdb::Status status = rocksdb::DB::Open(options, a.path, &db);
-
-				if (!status.ok()) {
-					logRocksDBError(status, "Restore");
-					a.done.sendError(statusToError(status));
-					return;
-				}
-
-				rocksdb::ExportImportFilesMetaData metaData = getMetaData(a.checkpoints[0]);
-				rocksdb::ImportColumnFamilyOptions importOptions;
-				importOptions.move_files = true;
-				status = db->CreateColumnFamilyWithImport(
-				    getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
-
-				if (!status.ok()) {
-					logRocksDBError(status, "Restore");
-					a.done.sendError(statusToError(status));
-				} else {
-					TraceEvent(SevInfo, "RocksDB").detail("Path", a.path).detail("Method", "Restore");
-					a.done.send(Void());
-				}
-			} else {
-				throw not_implemented();
-			}
-		}
 	};
 
 	struct Reader : IThreadPoolReceiver {
@@ -2043,6 +1946,171 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	}
 };
 
+void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
+	TraceEvent("RocksDBServeCheckpointBegin", id)
+	    .detail("MinVersion", a.request.version)
+	    .detail("Range", a.request.range.toString())
+	    .detail("Format", static_cast<int>(a.request.format))
+	    .detail("CheckpointDir", a.request.checkpointDir);
+
+	rocksdb::Checkpoint* checkpoint;
+	rocksdb::Status s = rocksdb::Checkpoint::Create(db, &checkpoint);
+	if (!s.ok()) {
+		logRocksDBError(s, "Checkpoint");
+		a.reply.sendError(statusToError(s));
+		return;
+	}
+
+	rocksdb::PinnableSlice value;
+	rocksdb::ReadOptions readOptions = getReadOptions();
+	s = db->Get(readOptions, cf, toSlice(persistVersion), &value);
+
+	if (!s.ok() && !s.IsNotFound()) {
+		logRocksDBError(s, "Checkpoint");
+		a.reply.sendError(statusToError(s));
+		return;
+	}
+
+	const Version version =
+	    s.IsNotFound() ? latestVersion : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
+
+	ASSERT(a.request.version == version || a.request.version == latestVersion);
+	TraceEvent(SevDebug, "RocksDBServeCheckpointVersion", id)
+	    .detail("CheckpointVersion", a.request.version)
+	    .detail("PersistVersion", version);
+
+	// TODO: set the range as the actual shard range.
+	CheckpointMetaData res(version, a.request.range, a.request.format, a.request.checkpointID);
+	const std::string& checkpointDir = abspath(a.request.checkpointDir);
+
+	if (a.request.format == RocksDBColumnFamily) {
+		rocksdb::ExportImportFilesMetaData* pMetadata;
+		platform::eraseDirectoryRecursive(checkpointDir);
+		s = checkpoint->ExportColumnFamily(cf, checkpointDir, &pMetadata);
+		if (!s.ok()) {
+			logRocksDBError(s, "ExportColumnFamily");
+			a.reply.sendError(statusToError(s));
+			return;
+		}
+
+		populateMetaData(&res, *pMetadata);
+		delete pMetadata;
+		TraceEvent("RocksDBServeCheckpointSuccess", id)
+		    .detail("CheckpointMetaData", res.toString())
+		    .detail("RocksDBCF", getRocksCF(res).toString());
+	} else if (a.request.format == RocksDB) {
+		platform::eraseDirectoryRecursive(checkpointDir);
+		uint64_t debugCheckpointSeq = -1;
+		s = checkpoint->CreateCheckpoint(checkpointDir, /*log_size_for_flush=*/0, &debugCheckpointSeq);
+		if (!s.ok()) {
+			logRocksDBError(s, "Checkpoint");
+			a.reply.sendError(statusToError(s));
+			return;
+		}
+
+		RocksDBCheckpoint rcp;
+		rcp.checkpointDir = checkpointDir;
+		rcp.sstFiles = platform::listFiles(checkpointDir, ".sst");
+		res.serializedCheckpoint = ObjectWriter::toValue(rcp, IncludeVersion());
+		TraceEvent("RocksDBCheckpointCreated", id)
+		    .detail("CheckpointVersion", a.request.version)
+		    .detail("RocksSequenceNumber", debugCheckpointSeq)
+		    .detail("CheckpointDir", checkpointDir);
+	} else {
+		throw not_implemented();
+	}
+
+	res.setState(CheckpointMetaData::Complete);
+	a.reply.send(res);
+}
+
+void RocksDBKeyValueStore::Writer::action(RestoreAction& a) {
+	TraceEvent("RocksDBRestoreBegin", id).detail("Path", a.path).detail("Checkpoints", describe(a.checkpoints));
+
+	ASSERT(db != nullptr);
+	ASSERT(!a.checkpoints.empty());
+
+	const CheckpointFormat format = a.checkpoints[0].getFormat();
+	for (int i = 1; i < a.checkpoints.size(); ++i) {
+		if (a.checkpoints[i].getFormat() != format) {
+			throw invalid_checkpoint_format();
+		}
+	}
+
+	rocksdb::Status status;
+	if (format == RocksDBColumnFamily) {
+		ASSERT_EQ(a.checkpoints.size(), 1);
+		TraceEvent("RocksDBServeRestoreCF", id)
+		    .detail("Path", a.path)
+		    .detail("Checkpoint", a.checkpoints[0].toString())
+		    .detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
+
+		if (cf != nullptr) {
+			ASSERT(db->DropColumnFamily(cf).ok());
+		}
+
+		rocksdb::ExportImportFilesMetaData metaData = getMetaData(a.checkpoints[0]);
+		rocksdb::ImportColumnFamilyOptions importOptions;
+		importOptions.move_files = true;
+		status = db->CreateColumnFamilyWithImport(
+		    getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
+
+		if (!status.ok()) {
+			logRocksDBError(status, "Restore");
+			a.done.sendError(statusToError(status));
+		} else {
+			TraceEvent(SevInfo, "RocksDBRestoreCFSuccess")
+			    .detail("Path", a.path)
+			    .detail("Checkpoint", a.checkpoints[0].toString());
+			a.done.send(Void());
+		}
+	} else if (format == RocksDB) {
+		if (cf == nullptr) {
+			status = db->CreateColumnFamily(getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, &cf);
+			TraceEvent("RocksDBServeRestoreRange", id)
+			    .detail("Path", a.path)
+			    .detail("Checkpoint", describe(a.checkpoints));
+			if (!status.ok()) {
+				logRocksDBError(status, "CreateColumnFamily");
+				a.done.sendError(statusToError(status));
+				return;
+			}
+		}
+
+		std::vector<std::string> sstFiles;
+		for (const auto& checkpoint : a.checkpoints) {
+			const RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
+			for (const auto& file : rocksCheckpoint.fetchedFiles) {
+				TraceEvent("RocksDBRestoreFile", id)
+				    .detail("Checkpoint", rocksCheckpoint.toString())
+				    .detail("File", file.toString());
+				sstFiles.push_back(file.path);
+			}
+		}
+
+		if (!sstFiles.empty()) {
+			rocksdb::IngestExternalFileOptions ingestOptions;
+			ingestOptions.move_files = true;
+			ingestOptions.write_global_seqno = false;
+			ingestOptions.verify_checksums_before_ingest = true;
+			status = db->IngestExternalFile(cf, sstFiles, ingestOptions);
+			if (!status.ok()) {
+				logRocksDBError(status, "IngestExternalFile", SevWarnAlways);
+				a.done.sendError(statusToError(status));
+				return;
+			}
+		} else {
+			TraceEvent(SevDebug, "RocksDBServeRestoreEmptyRange", id)
+			    .detail("Path", a.path)
+			    .detail("Checkpoint", describe(a.checkpoints));
+		}
+		TraceEvent("RocksDBServeRestoreEnd", id).detail("Path", a.path).detail("Checkpoint", describe(a.checkpoints));
+		a.done.send(Void());
+	} else {
+		throw not_implemented();
+	}
+}
+
 } // namespace
 
 #endif // SSD_ROCKSDB_EXPERIMENTAL
@@ -2155,7 +2223,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBReopen") {
 	return Void();
 }
 
-TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
+TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreColumnFamily") {
 	state std::string cwd = platform::getWorkingDirectory() + "/";
 	state std::string rocksDBTestDir = "rocksdb-kvstore-br-test-db";
 	platform::eraseDirectoryRecursive(rocksDBTestDir);
@@ -2169,6 +2237,13 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
 	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
 
+	state std::string rocksDBRestoreDir = "rocksdb-kvstore-br-restore-db";
+	platform::eraseDirectoryRecursive(rocksDBRestoreDir);
+
+	state IKeyValueStore* kvStoreCopy =
+	    new RocksDBKeyValueStore(rocksDBRestoreDir, deterministicRandom()->randomUniqueID());
+	wait(kvStoreCopy->init());
+
 	platform::eraseDirectoryRecursive("checkpoint");
 	state std::string checkpointDir = cwd + "checkpoint";
 
@@ -2176,12 +2251,6 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 	    latestVersion, allKeys, RocksDBColumnFamily, deterministicRandom()->randomUniqueID(), checkpointDir);
 	CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
 
-	state std::string rocksDBRestoreDir = "rocksdb-kvstore-br-restore-db";
-	platform::eraseDirectoryRecursive(rocksDBRestoreDir);
-
-	state IKeyValueStore* kvStoreCopy =
-	    new RocksDBKeyValueStore(rocksDBRestoreDir, deterministicRandom()->randomUniqueID());
-
 	std::vector<CheckpointMetaData> checkpoints;
 	checkpoints.push_back(metaData);
 	wait(kvStoreCopy->restore(checkpoints));
@@ -2202,11 +2271,52 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 	return Void();
 }
 
-TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
+TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreKeyValues") {
-	// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
+	state std::string cwd = platform::getWorkingDirectory() + "/";
-	// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
+	state std::string rocksDBTestDir = "rocksdb-kvstore-brsst-test-db";
-	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 184);
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
-	ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
+	state IKeyValueStore* kvStore = new RocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	kvStore->set({ LiteralStringRef("foo"), LiteralStringRef("bar") });
+	wait(kvStore->commit(false));
+	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
+	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+
+	platform::eraseDirectoryRecursive("checkpoint");
+	std::string checkpointDir = cwd + "checkpoint";
+
+	CheckpointRequest request(latestVersion, allKeys, RocksDB, deterministicRandom()->randomUniqueID(), checkpointDir);
+	CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
+
+	state ICheckpointReader* cpReader = newCheckpointReader(metaData, deterministicRandom()->randomUniqueID());
+	wait(cpReader->init(BinaryWriter::toValue(KeyRangeRef("foo"_sr, "foobar"_sr), IncludeVersion())));
+	loop {
+		try {
+			state RangeResult res =
+			    wait(cpReader->nextKeyValues(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
+			state int i = 0;
+			for (; i < res.size(); ++i) {
+				Optional<Value> val = wait(kvStore->readValue(res[i].key));
+				ASSERT(val.present() && val.get() == res[i].value);
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_end_of_stream) {
+				break;
+			} else {
+				TraceEvent(SevError, "TestFailed").error(e);
+			}
+		}
+	}
+
+	std::vector<Future<Void>> closes;
+	closes.push_back(cpReader->close());
+	closes.push_back(kvStore->onClosed());
+	kvStore->close();
+	wait(waitForAll(closes));
+
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
 	return Void();
 }
 

fdbserver/KmsConnectorInterface.h

@@ -101,16 +101,18 @@ struct KmsConnLookupEKsByKeyIdsRep {
 struct KmsConnLookupEKsByKeyIdsReq {
 	constexpr static FileIdentifier file_identifier = 6913396;
 	std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> encryptKeyIds;
+	Optional<UID> debugId;
 	ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
 
 	KmsConnLookupEKsByKeyIdsReq() {}
 	explicit KmsConnLookupEKsByKeyIdsReq(
-	    const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds)
+	    const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds,
-	  : encryptKeyIds(keyIds) {}
+	    Optional<UID> dbgId)
+	  : encryptKeyIds(keyIds), debugId(dbgId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, encryptKeyIds, reply);
+		serializer(ar, encryptKeyIds, debugId, reply);
 	}
 };
 
@@ -130,14 +132,16 @@ struct KmsConnLookupEKsByDomainIdsRep {
 struct KmsConnLookupEKsByDomainIdsReq {
 	constexpr static FileIdentifier file_identifier = 9918682;
 	std::vector<EncryptCipherDomainId> encryptDomainIds;
+	Optional<UID> debugId;
 	ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
 
 	KmsConnLookupEKsByDomainIdsReq() {}
-	explicit KmsConnLookupEKsByDomainIdsReq(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
+	explicit KmsConnLookupEKsByDomainIdsReq(const std::vector<EncryptCipherDomainId>& ids, Optional<UID> dbgId)
+	  : encryptDomainIds(ids), debugId(dbgId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, encryptDomainIds, reply);
+		serializer(ar, encryptDomainIds, debugId, reply);
 	}
 };
 

fdbserver/ProxyCommitData.actor.h

@@ -104,7 +104,8 @@ struct ProxyStats {
 	explicit ProxyStats(UID id,
 	                    NotifiedVersion* pVersion,
 	                    NotifiedVersion* pCommittedVersion,
-	                    int64_t* commitBatchesMemBytesCountPtr)
+	                    int64_t* commitBatchesMemBytesCountPtr,
+	                    std::map<TenantName, TenantMapEntry>* pTenantMap)
 	  : cc("ProxyStats", id.toString()), txnCommitIn("TxnCommitIn", cc),
 	    txnCommitVersionAssigned("TxnCommitVersionAssigned", cc), txnCommitResolving("TxnCommitResolving", cc),
 	    txnCommitResolved("TxnCommitResolved", cc), txnCommitOut("TxnCommitOut", cc),
@@ -160,6 +161,7 @@ struct ProxyStats {
 		specialCounter(cc, "CommitBatchesMemBytesCount", [commitBatchesMemBytesCountPtr]() {
 			return *commitBatchesMemBytesCountPtr;
 		});
+		specialCounter(cc, "NumTenants", [pTenantMap]() { return pTenantMap ? pTenantMap->size() : 0; });
 		specialCounter(cc, "MaxCompute", [this]() { return this->getAndResetMaxCompute(); });
 		specialCounter(cc, "MinCompute", [this]() { return this->getAndResetMinCompute(); });
 		logger = traceCounters("ProxyMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ProxyMetrics");
@@ -169,6 +171,7 @@ struct ProxyStats {
 struct ProxyCommitData {
 	UID dbgid;
 	int64_t commitBatchesMemBytesCount;
+	std::map<TenantName, TenantMapEntry> tenantMap;
 	ProxyStats stats;
 	MasterInterface master;
 	std::vector<ResolverInterface> resolvers;
@@ -226,8 +229,6 @@ struct ProxyCommitData {
 	UIDTransactionTagMap<TransactionCommitCostEstimation> ssTrTagCommitCost;
 	double lastMasterReset;
 	double lastResolverReset;
-
-	std::map<TenantName, TenantMapEntry> tenantMap;
 	int localTLogCount = -1;
 
 	// The tag related to a storage server rarely change, so we keep a vector of tags for each key range to be slightly
@@ -289,11 +290,12 @@ struct ProxyCommitData {
 	                Reference<AsyncVar<ServerDBInfo> const> db,
 	                bool firstProxy)
 	  : dbgid(dbgid), commitBatchesMemBytesCount(0),
-	    stats(dbgid, &version, &committedVersion, &commitBatchesMemBytesCount), master(master), logAdapter(nullptr),
+	    stats(dbgid, &version, &committedVersion, &commitBatchesMemBytesCount, &tenantMap), master(master),
-	    txnStateStore(nullptr), committedVersion(recoveryTransactionVersion), minKnownCommittedVersion(0), version(0),
+	    logAdapter(nullptr), txnStateStore(nullptr), committedVersion(recoveryTransactionVersion),
-	    lastVersionTime(0), commitVersionRequestNumber(1), mostRecentProcessedRequestNumber(0), firstProxy(firstProxy),
+	    minKnownCommittedVersion(0), version(0), lastVersionTime(0), commitVersionRequestNumber(1),
-	    lastCoalesceTime(0), locked(false), commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN),
+	    mostRecentProcessedRequestNumber(0), firstProxy(firstProxy), lastCoalesceTime(0), locked(false),
-	    localCommitBatchesStarted(0), getConsistentReadVersion(getConsistentReadVersion), commit(commit),
+	    commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), localCommitBatchesStarted(0),
+	    getConsistentReadVersion(getConsistentReadVersion), commit(commit),
 	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db),
 	    singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")), lastTxsPop(0), popRemoteTxs(false),
 	    lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0),

fdbserver/QuietDatabase.actor.cpp

@@ -20,6 +20,7 @@
 
 #include <cinttypes>
 #include <vector>
+#include <type_traits>
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/SystemData.h"
@@ -669,6 +670,60 @@ ACTOR Future<Void> reconfigureAfter(Database cx,
 	return Void();
 }
 
+struct QuietDatabaseChecker {
+	double start = now();
+	constexpr static double maxDDRunTime = 1000.0;
+
+	struct Impl {
+		double start;
+		std::string const& phase;
+		std::vector<std::string> failReasons;
+
+		Impl(double start, const std::string& phase) : start(start), phase(phase) {}
+
+		template <class T, class Comparison = std::less_equal<>>
+		Impl& add(BaseTraceEvent& evt,
+		          const char* name,
+		          T value,
+		          T expected,
+		          Comparison const& cmp = std::less_equal<>()) {
+			std::string k = fmt::format("{}Gate", name);
+			evt.detail(name, value).detail(k.c_str(), expected);
+			if (!cmp(value, expected)) {
+				failReasons.push_back(name);
+			}
+			return *this;
+		}
+
+		bool success() {
+			bool timedOut = now() - start > maxDDRunTime;
+			if (!failReasons.empty()) {
+				std::string traceMessage = fmt::format("QuietDatabase{}Fail", phase);
+				std::string reasons = fmt::format("{}", fmt::join(failReasons, ", "));
+				TraceEvent(timedOut ? SevError : SevWarnAlways, traceMessage.c_str())
+				    .detail("Reasons", reasons)
+				    .detail("FailedAfter", now() - start)
+				    .detail("Timeout", maxDDRunTime);
+				if (timedOut) {
+					// this bool is just created to make the assertion more readable
+					bool ddGotStuck = true;
+					// This assertion is here to make the test fail more quickly. If quietDatabase takes this
+					// long without completing, we can assume that the test will eventually time out. However,
+					// time outs are more annoying to debug. This will hopefully be easier to track down.
+					ASSERT(!ddGotStuck || !g_network->isSimulated());
+				}
+				return false;
+			}
+			return true;
+		}
+	};
+
+	Impl startIteration(std::string const& phase) const {
+		Impl res(start, phase);
+		return res;
+	}
+};
+
 // Waits until a database quiets down (no data in flight, small tlog queue, low SQ, no active data distribution). This
 // requires the database to be available and healthy in order to succeed.
 ACTOR Future<Void> waitForQuietDatabase(Database cx,
@@ -680,6 +735,7 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
                                         int64_t maxDataDistributionQueueSize = 0,
                                         int64_t maxPoppedVersionLag = 30e6,
                                         int64_t maxVersionOffset = 1e6) {
+	state QuietDatabaseChecker checker;
 	state Future<Void> reconfig =
 	    reconfigureAfter(cx, 100 + (deterministicRandom()->random01() * 100), dbInfo, "QuietDatabase");
 	state Future<int64_t> dataInFlight;
@@ -732,35 +788,26 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 			     success(teamCollectionValid) && success(storageQueueSize) && success(dataDistributionActive) &&
 			     success(storageServersRecruiting) && success(versionOffset));
 
-			TraceEvent(("QuietDatabase" + phase).c_str())
-			    .detail("DataInFlight", dataInFlight.get())
-			    .detail("DataInFlightGate", dataInFlightGate)
-			    .detail("MaxTLogQueueSize", tLogQueueInfo.get().first)
-			    .detail("MaxTLogQueueGate", maxTLogQueueGate)
-			    .detail("MaxTLogPoppedVersionLag", tLogQueueInfo.get().second)
-			    .detail("MaxTLogPoppedVersionLagGate", maxPoppedVersionLag)
-			    .detail("DataDistributionQueueSize", dataDistributionQueueSize.get())
-			    .detail("DataDistributionQueueSizeGate", maxDataDistributionQueueSize)
-			    .detail("TeamCollectionValid", teamCollectionValid.get())
-			    .detail("MaxStorageQueueSize", storageQueueSize.get())
-			    .detail("MaxStorageServerQueueGate", maxStorageServerQueueGate)
-			    .detail("DataDistributionActive", dataDistributionActive.get())
-			    .detail("StorageServersRecruiting", storageServersRecruiting.get())
-			    .detail("RecoveryCount", dbInfo->get().recoveryCount)
-			    .detail("VersionOffset", versionOffset.get())
-			    .detail("NumSuccesses", numSuccesses);
-
 			maxVersionOffset += dbInfo->get().recoveryCount * SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT;
-			if (dataInFlight.get() > dataInFlightGate || tLogQueueInfo.get().first > maxTLogQueueGate ||
-			    tLogQueueInfo.get().second > maxPoppedVersionLag ||
-			    dataDistributionQueueSize.get() > maxDataDistributionQueueSize ||
-			    storageQueueSize.get() > maxStorageServerQueueGate || !dataDistributionActive.get() ||
-			    storageServersRecruiting.get() || versionOffset.get() > maxVersionOffset ||
-			    !teamCollectionValid.get()) {
 
-				wait(delay(1.0));
+			auto check = checker.startIteration(phase);
-				numSuccesses = 0;
+
-			} else {
+			std::string evtType = "QuietDatabase" + phase;
+			TraceEvent evt(evtType.c_str());
+			check.add(evt, "DataInFlight", dataInFlight.get(), dataInFlightGate)
+			    .add(evt, "MaxTLogQueueSize", tLogQueueInfo.get().first, maxTLogQueueGate)
+			    .add(evt, "MaxTLogPoppedVersionLag", tLogQueueInfo.get().second, maxPoppedVersionLag)
+			    .add(evt, "DataDistributionQueueSize", dataDistributionQueueSize.get(), maxDataDistributionQueueSize)
+			    .add(evt, "TeamCollectionValid", teamCollectionValid.get(), true, std::equal_to<>())
+			    .add(evt, "MaxStorageQueueSize", storageQueueSize.get(), maxStorageServerQueueGate)
+			    .add(evt, "DataDistributionActive", dataDistributionActive.get(), true, std::equal_to<>())
+			    .add(evt, "StorageServersRecruiting", storageServersRecruiting.get(), false, std::equal_to<>())
+			    .add(evt, "VersionOffset", versionOffset.get(), maxVersionOffset);
+
+			evt.detail("RecoveryCount", dbInfo->get().recoveryCount).detail("NumSuccesses", numSuccesses);
+			evt.log();
+
+			if (check.success()) {
 				if (++numSuccesses == 3) {
 					auto msg = "QuietDatabase" + phase + "Done";
 					TraceEvent(msg.c_str()).log();
@@ -768,6 +815,9 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 				} else {
 					wait(delay(g_network->isSimulated() ? 2.0 : 30.0));
 				}
+			} else {
+				wait(delay(1.0));
+				numSuccesses = 0;
 			}
 		} catch (Error& e) {
 			TraceEvent(("QuietDatabase" + phase + "Error").c_str()).errorUnsuppressed(e);

fdbserver/RESTKmsConnector.actor.cpp

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/RESTKmsConnector.actor.h"
+#include "fdbserver/RESTKmsConnector.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/rapidjson/document.h"
@@ -54,13 +54,15 @@ const char* BASE_CIPHER_TAG = "baseCipher";
 const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details";
 const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id";
 const char* ERROR_TAG = "error";
-const char* ERROR_DETAIL_TAG = "details";
+const char* ERROR_MSG_TAG = "errMsg";
+const char* ERROR_CODE_TAG = "errCode";
 const char* KMS_URLS_TAG = "kms_urls";
 const char* QUERY_MODE_TAG = "query_mode";
 const char* REFRESH_KMS_URLS_TAG = "refresh_kms_urls";
 const char* VALIDATION_TOKENS_TAG = "validation_tokens";
 const char* VALIDATION_TOKEN_NAME_TAG = "token_name";
 const char* VALIDATION_TOKEN_VALUE_TAG = "token_value";
+const char* DEBUG_UID_TAG = "debug_uid";
 
 const char* TOKEN_NAME_FILE_SEP = "#";
 const char* TOKEN_TUPLE_SEP = ",";
@@ -280,9 +282,10 @@ void parseKmsResponse(Reference<RESTKmsConnectorCtx> ctx,
 	//   "kms_urls" : [
 	//         "url1", "url2", ...
 	//   ],
-	//	 "error" : {
+	//	 "error" : {					// Optional, populated by the KMS, if present, rest of payload is ignored.
-	//		"details": <details>
+	//		"errMsg" : <message>
-	// 	  }  					// Optional, populated by the KMS, if present, rest of payload is ignored.
+	//		"errCode": <code>
+	// 	  }
 	// }
 
 	if (resp->code != HTTP::HTTP_STATUS_CODE_OK) {
@@ -295,12 +298,26 @@ void parseKmsResponse(Reference<RESTKmsConnectorCtx> ctx,
 
 	// Check if response has error
 	if (doc.HasMember(ERROR_TAG)) {
-		if (doc[ERROR_TAG].HasMember(ERROR_DETAIL_TAG) && doc[ERROR_TAG][ERROR_DETAIL_TAG].IsString()) {
+		Standalone<StringRef> errMsgRef;
-			Standalone<StringRef> errRef = makeString(doc[ERROR_TAG][ERROR_DETAIL_TAG].GetStringLength());
+		Standalone<StringRef> errCodeRef;
-			memcpy(mutateString(errRef),
+
-			       doc[ERROR_TAG][ERROR_DETAIL_TAG].GetString(),
+		if (doc[ERROR_TAG].HasMember(ERROR_MSG_TAG) && doc[ERROR_TAG][ERROR_MSG_TAG].IsString()) {
-			       doc[ERROR_TAG][ERROR_DETAIL_TAG].GetStringLength());
+			errMsgRef = makeString(doc[ERROR_TAG][ERROR_MSG_TAG].GetStringLength());
-			TraceEvent("KMSErrorResponse", ctx->uid).detail("ErrorDetails", errRef.toString());
+			memcpy(mutateString(errMsgRef),
+			       doc[ERROR_TAG][ERROR_MSG_TAG].GetString(),
+			       doc[ERROR_TAG][ERROR_MSG_TAG].GetStringLength());
+		}
+		if (doc[ERROR_TAG].HasMember(ERROR_CODE_TAG) && doc[ERROR_TAG][ERROR_CODE_TAG].IsString()) {
+			errMsgRef = makeString(doc[ERROR_TAG][ERROR_CODE_TAG].GetStringLength());
+			memcpy(mutateString(errMsgRef),
+			       doc[ERROR_TAG][ERROR_CODE_TAG].GetString(),
+			       doc[ERROR_TAG][ERROR_CODE_TAG].GetStringLength());
+		}
+
+		if (!errCodeRef.empty() || !errMsgRef.empty()) {
+			TraceEvent("KMSErrorResponse", ctx->uid)
+			    .detail("ErrorMsg", errMsgRef.empty() ? "" : errMsgRef.toString())
+			    .detail("ErrorCode", errCodeRef.empty() ? "" : errCodeRef.toString());
 		} else {
 			TraceEvent("KMSErrorResponse_EmptyDetails", ctx->uid).log();
 		}
@@ -397,6 +414,20 @@ void addRefreshKmsUrlsSectionToJsonDoc(Reference<RESTKmsConnectorCtx> ctx,
 	doc.AddMember(key, refreshUrls, doc.GetAllocator());
 }
 
+void addDebugUidSectionToJsonDoc(Reference<RESTKmsConnectorCtx> ctx, rapidjson::Document& doc, Optional<UID> dbgId) {
+	if (!dbgId.present()) {
+		// Debug id not present; do nothing
+		return;
+	}
+	rapidjson::Value key(DEBUG_UID_TAG, doc.GetAllocator());
+	rapidjson::Value debugIdVal;
+	const std::string dbgIdStr = dbgId.get().toString();
+	debugIdVal.SetString(dbgIdStr.c_str(), dbgIdStr.size(), doc.GetAllocator());
+
+	// Append 'debug_uid' object to the parent document
+	doc.AddMember(key, debugIdVal, doc.GetAllocator());
+}
+
 StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
                                             const KmsConnLookupEKsByKeyIdsReq& req,
                                             const bool refreshKmsUrls,
@@ -424,6 +455,7 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
 	//     }
 	//   ]
 	//   "refresh_kms_urls" = 1/0
+	//   "debug_uid" = <uid-string>   // Optional debug info to trace requests across FDB <--> KMS
 	// }
 
 	rapidjson::Document doc;
@@ -458,9 +490,12 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
 	// Append 'validation_tokens' as json array
 	addValidationTokensSectionToJsonDoc(ctx, doc);
 
-	// Append "refresh_kms_urls'
+	// Append 'refresh_kms_urls'
 	addRefreshKmsUrlsSectionToJsonDoc(ctx, doc, refreshKmsUrls);
 
+	// Append 'debug_uid' section if needed
+	addDebugUidSectionToJsonDoc(ctx, doc, req.debugId);
+
 	// Serialize json to string
 	rapidjson::StringBuffer sb;
 	rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@@ -574,6 +609,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
 	//     }
 	//   ]
 	//   "refresh_kms_urls" = 1/0
+	//   "debug_uid" = <uid-string>     // Optional debug info to trace requests across FDB <--> KMS
 	// }
 
 	rapidjson::Document doc;
@@ -604,6 +640,9 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
 	// Append 'refresh_kms_urls'
 	addRefreshKmsUrlsSectionToJsonDoc(ctx, doc, refreshKmsUrls);
 
+	// Append 'debug_uid' section if needed
+	addDebugUidSectionToJsonDoc(ctx, doc, req.debugId);
+
 	// Serialize json to string
 	rapidjson::StringBuffer sb;
 	rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@@ -1007,13 +1046,16 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
 	}
 
 	bool refreshKmsUrls = deterministicRandom()->randomInt(0, 100) < 50;
+	if (deterministicRandom()->randomInt(0, 100) < 40) {
+		req.debugId = deterministicRandom()->randomUniqueID();
+	}
 
 	StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, arena);
-	TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(10000).detail("JsonReqStr", requestBodyRef.toString());
+	TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(100000).detail("JsonReqStr", requestBodyRef.toString());
 	Reference<HTTP::Response> httpResp = makeReference<HTTP::Response>();
 	httpResp->code = HTTP::HTTP_STATUS_CODE_OK;
 	getFakeKmsResponse(requestBodyRef, true, httpResp);
-	TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(10000).detail("HttpRespStr", httpResp->content);
+	TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(100000).detail("HttpRespStr", httpResp->content);
 
 	std::vector<EncryptCipherKeyDetails> cipherDetails;
 	parseKmsResponse(ctx, httpResp, &arena, &cipherDetails);
@@ -1168,7 +1210,7 @@ void testKMSErrorResponse(Reference<RESTKmsConnectorCtx> ctx) {
 	rapidjson::Value errorTag(rapidjson::kObjectType);
 
 	// Add 'error_detail'
-	rapidjson::Value eKey(ERROR_DETAIL_TAG, doc.GetAllocator());
+	rapidjson::Value eKey(ERROR_MSG_TAG, doc.GetAllocator());
 	rapidjson::Value detailInfo;
 	detailInfo.SetString("Foo is always bad", doc.GetAllocator());
 	errorTag.AddMember(eKey, detailInfo, doc.GetAllocator());

fdbserver/RESTKmsConnector.h

@@ -18,14 +18,10 @@
  * limitations under the License.
  */
 
+#ifndef REST_KMS_CONNECTOR_H
+#define REST_KMS_CONNECTOR_H
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTKMSCONNECTOR_ACTOR_G_H)
-#define FDBSERVER_RESTKMSCONNECTOR_ACTOR_G_H
-#include "fdbserver/RESTKmsConnector.actor.g.h"
-#elif !defined(FDBSERVER_RESTKMSCONNECTOR_ACTOR_H)
-#define FDBSERVER_RESTKMSCONNECTOR_ACTOR_H
-
 #include "fdbserver/KmsConnector.h"
 
 class RESTKmsConnector : public KmsConnector {

fdbserver/RestoreLoader.actor.cpp

@@ -27,7 +27,7 @@
 #include "fdbserver/RestoreLoader.actor.h"
 #include "fdbserver/RestoreRoleCommon.actor.h"
 #include "fdbserver/MutationTracking.h"
-#include "fdbserver/StorageMetrics.actor.h"
+#include "fdbserver/StorageMetrics.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 

fdbserver/RocksDBCheckpointUtils.actor.cpp

@@ -20,32 +20,394 @@
 
 #include "fdbserver/RocksDBCheckpointUtils.actor.h"
 
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+#include <rocksdb/db.h>
+#include <rocksdb/env.h>
+#include <rocksdb/options.h>
+#include <rocksdb/slice.h>
+#include <rocksdb/slice_transform.h>
+#include <rocksdb/types.h>
+#include <rocksdb/version.h>
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/StorageCheckpoint.h"
+#include "fdbserver/CoroFlow.h"
+#include "fdbserver/Knobs.h"
+#include "flow/IThreadPool.h"
+#include "flow/ThreadHelper.actor.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
 
 #include "flow/actorcompiler.h" // has to be last include
 
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+// Enforcing rocksdb version to be 6.22.1 or greater.
+static_assert(ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22 && ROCKSDB_PATCH >= 1,
+              "Unsupported rocksdb version. Update the rocksdb to at least 6.22.1 version");
+
 namespace {
 
+using DB = rocksdb::DB*;
+using CF = rocksdb::ColumnFamilyHandle*;
+
+const KeyRef persistVersion = "\xff\xffVersion"_sr;
+
+rocksdb::Slice toSlice(StringRef s) {
+	return rocksdb::Slice(reinterpret_cast<const char*>(s.begin()), s.size());
+}
+
+StringRef toStringRef(rocksdb::Slice s) {
+	return StringRef(reinterpret_cast<const uint8_t*>(s.data()), s.size());
+}
+
+rocksdb::ColumnFamilyOptions getCFOptions() {
+	rocksdb::ColumnFamilyOptions options;
+	return options;
+}
+
+rocksdb::Options getOptions() {
+	rocksdb::Options options({}, getCFOptions());
+	options.create_if_missing = false;
+	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
+	return options;
+}
+
+// Set some useful defaults desired for all reads.
+rocksdb::ReadOptions getReadOptions() {
+	rocksdb::ReadOptions options;
+	options.background_purge_on_iterator_cleanup = true;
+	return options;
+}
+
+void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
+	auto level = status.IsTimedOut() ? SevWarn : SevError;
+	TraceEvent e(level, "RocksDBCheckpointReaderError");
+	e.detail("Error", status.ToString()).detail("Method", method).detail("RocksDBSeverity", status.severity());
+	if (status.IsIOError()) {
+		e.detail("SubCode", status.subcode());
+	}
+}
+
+Error statusToError(const rocksdb::Status& s) {
+	if (s.IsIOError()) {
+		return io_error();
+	} else if (s.IsTimedOut()) {
+		return transaction_too_old();
+	} else {
+		return unknown_error();
+	}
+}
+
+// RocksDBCheckpointReader reads a RocksDB checkpoint, and returns the key-value pairs via nextKeyValues.
 class RocksDBCheckpointReader : public ICheckpointReader {
 public:
-	RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
+	RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID);
+
+	Future<Void> init(StringRef token) override;
+
+	Future<RangeResult> nextKeyValues(const int rowLimit, const int byteLimit) override;
+
+	Future<Standalone<StringRef>> nextChunk(const int byteLimit) { throw not_implemented(); }
+
+	Future<Void> close() { return doClose(this); }
+
+private:
+	struct Reader : IThreadPoolReceiver {
+		struct OpenAction : TypedAction<Reader, OpenAction> {
+			OpenAction(std::string path, KeyRange range, Version version)
+			  : path(std::move(path)), range(range), version(version) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			const std::string path;
+			const KeyRange range;
+			const Version version;
+			ThreadReturnPromise<Void> done;
+		};
+
+		struct CloseAction : TypedAction<Reader, CloseAction> {
+			CloseAction(std::string path, bool deleteOnClose) : path(path), deleteOnClose(deleteOnClose) {}
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			std::string path;
+			bool deleteOnClose;
+			ThreadReturnPromise<Void> done;
+		};
+
+		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
+			ReadRangeAction(int rowLimit, int byteLimit)
+			  : rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
+
+			const int rowLimit, byteLimit;
+			const double startTime;
+			ThreadReturnPromise<RangeResult> result;
+		};
+
+		explicit Reader(DB& db);
+		~Reader() override {}
+
+		void init() override {}
+
+		void action(OpenAction& a);
+
+		void action(CloseAction& a);
+
+		void action(ReadRangeAction& a);
+
+		DB& db;
+		CF cf;
+		Key begin;
+		Key end;
+		double readRangeTimeout;
+		std::unique_ptr<rocksdb::Iterator> cursor;
+	};
+
+	ACTOR static Future<Void> doClose(RocksDBCheckpointReader* self);
+
+	DB db = nullptr;
+	std::string path;
+	const UID id;
+	Version version;
+	Reference<IThreadPool> readThreads;
+	Future<Void> openFuture;
+};
+
+RocksDBCheckpointReader::RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
+  : id(logID), version(checkpoint.version) {
+	RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
+	this->path = rocksCheckpoint.checkpointDir;
+	if (g_network->isSimulated()) {
+		readThreads = CoroThreadPool::createThreadPool();
+	} else {
+		readThreads = createGenericThreadPool();
+	}
+	readThreads->addThread(new Reader(db), "fdb-rocks-rd");
+}
+
+Future<Void> RocksDBCheckpointReader::init(StringRef token) {
+	if (openFuture.isValid()) {
+		return openFuture;
+	}
+
+	KeyRange range = BinaryReader::fromStringRef<KeyRange>(token, IncludeVersion());
+	auto a = std::make_unique<Reader::OpenAction>(this->path, range, this->version);
+	openFuture = a->done.getFuture();
+	readThreads->post(a.release());
+	return openFuture;
+}
+
+Future<RangeResult> RocksDBCheckpointReader::nextKeyValues(const int rowLimit, const int byteLimit) {
+	auto a = std::make_unique<Reader::ReadRangeAction>(rowLimit, byteLimit);
+	auto res = a->result.getFuture();
+	readThreads->post(a.release());
+	return res;
+}
+
+RocksDBCheckpointReader::Reader::Reader(DB& db) : db(db), cf(nullptr) {
+	if (g_network->isSimulated()) {
+		// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
+		// very high load and single read thread cannot process all the load within the timeouts.
+		readRangeTimeout = 5 * 60;
+	} else {
+		readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
+	}
+}
+
+void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::OpenAction& a) {
+	ASSERT(cf == nullptr);
+
+	std::vector<std::string> columnFamilies;
+	rocksdb::Options options = getOptions();
+	rocksdb::Status status = rocksdb::DB::ListColumnFamilies(options, a.path, &columnFamilies);
+	if (std::find(columnFamilies.begin(), columnFamilies.end(), "default") == columnFamilies.end()) {
+		columnFamilies.push_back("default");
+	}
+
+	rocksdb::ColumnFamilyOptions cfOptions = getCFOptions();
+	std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+	for (const std::string& name : columnFamilies) {
+		descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, cfOptions });
+	}
+
+	std::vector<rocksdb::ColumnFamilyHandle*> handles;
+	status = rocksdb::DB::OpenForReadOnly(options, a.path, descriptors, &handles, &db);
+
+	if (!status.ok()) {
+		logRocksDBError(status, "OpenForReadOnly");
+		a.done.sendError(statusToError(status));
+		return;
+	}
+
+	for (rocksdb::ColumnFamilyHandle* handle : handles) {
+		if (handle->GetName() == SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY) {
+			cf = handle;
+			break;
+		}
+	}
+
+	ASSERT(db != nullptr && cf != nullptr);
+
+	begin = a.range.begin;
+	end = a.range.end;
+
+	TraceEvent(SevInfo, "RocksDBCheckpointReaderInit")
+	    .detail("Path", a.path)
+	    .detail("Method", "OpenForReadOnly")
+	    .detail("ColumnFamily", cf->GetName())
+	    .detail("Begin", begin)
+	    .detail("End", end);
+
+	rocksdb::PinnableSlice value;
+	rocksdb::ReadOptions readOptions = getReadOptions();
+	status = db->Get(readOptions, cf, toSlice(persistVersion), &value);
+
+	if (!status.ok() && !status.IsNotFound()) {
+		logRocksDBError(status, "Checkpoint");
+		a.done.sendError(statusToError(status));
+		return;
+	}
+
+	const Version version =
+	    status.IsNotFound() ? latestVersion : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
+
+	ASSERT(version == a.version);
+
+	cursor = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(readOptions, cf));
+	cursor->Seek(toSlice(begin));
+
+	a.done.send(Void());
+}
+
+void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::CloseAction& a) {
+	if (db == nullptr) {
+		a.done.send(Void());
+		return;
+	}
+
+	rocksdb::Status s = db->Close();
+	if (!s.ok()) {
+		logRocksDBError(s, "Close");
+	}
+
+	if (a.deleteOnClose) {
+		std::set<std::string> columnFamilies{ "default" };
+		columnFamilies.insert(SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY);
+		std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+		for (const std::string& name : columnFamilies) {
+			descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, getCFOptions() });
+		}
+		s = rocksdb::DestroyDB(a.path, getOptions(), descriptors);
+		if (!s.ok()) {
+			logRocksDBError(s, "Destroy");
+		} else {
+			TraceEvent("RocksDBCheckpointReader").detail("Path", a.path).detail("Method", "Destroy");
+		}
+	}
+
+	TraceEvent("RocksDBCheckpointReader").detail("Path", a.path).detail("Method", "Close");
+	a.done.send(Void());
+}
+
+void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::ReadRangeAction& a) {
+	const double readBeginTime = timer_monotonic();
+
+	if (readBeginTime - a.startTime > readRangeTimeout) {
+		TraceEvent(SevWarn, "RocksDBCheckpointReaderError")
+		    .detail("Error", "Read range request timedout")
+		    .detail("Method", "ReadRangeAction")
+		    .detail("Timeout value", readRangeTimeout);
+		a.result.sendError(timed_out());
+		return;
+	}
+
+	RangeResult result;
+	if (a.rowLimit == 0 || a.byteLimit == 0) {
+		a.result.send(result);
+		return;
+	}
+
+	// For now, only forward scan is supported.
+	ASSERT(a.rowLimit > 0);
+
+	int accumulatedBytes = 0;
+	rocksdb::Status s;
+	while (cursor->Valid() && toStringRef(cursor->key()) < end) {
+		KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
+		accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
+		result.push_back_deep(result.arena(), kv);
+		cursor->Next();
+		if (result.size() >= a.rowLimit || accumulatedBytes >= a.byteLimit) {
+			break;
+		}
+		if (timer_monotonic() - a.startTime > readRangeTimeout) {
+			TraceEvent(SevWarn, "RocksDBCheckpointReaderError")
+			    .detail("Error", "Read range request timedout")
+			    .detail("Method", "ReadRangeAction")
+			    .detail("Timeout value", readRangeTimeout);
+			a.result.sendError(transaction_too_old());
+			delete (cursor.release());
+			return;
+		}
+	}
+
+	s = cursor->status();
+
+	if (!s.ok()) {
+		logRocksDBError(s, "ReadRange");
+		a.result.sendError(statusToError(s));
+		delete (cursor.release());
+		return;
+	}
+
+	if (result.empty()) {
+		delete (cursor.release());
+		a.result.sendError(end_of_stream());
+	} else {
+		a.result.send(result);
+	}
+}
+
+ACTOR Future<Void> RocksDBCheckpointReader::doClose(RocksDBCheckpointReader* self) {
+	if (self == nullptr)
+		return Void();
+
+	auto a = new RocksDBCheckpointReader::Reader::CloseAction(self->path, false);
+	auto f = a->done.getFuture();
+	self->readThreads->post(a);
+	wait(f);
+
+	if (self != nullptr) {
+		wait(self->readThreads->stop());
+	}
+
+	if (self != nullptr) {
+		delete self;
+	}
+
+	return Void();
+}
+
+// RocksDBCFCheckpointReader reads an exported RocksDB Column Family checkpoint, and returns the serialized
+// checkpoint via nextChunk.
+class RocksDBCFCheckpointReader : public ICheckpointReader {
+public:
+	RocksDBCFCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
 	  : checkpoint_(checkpoint), id_(logID), file_(Reference<IAsyncFile>()), offset_(0) {}
 
 	Future<Void> init(StringRef token) override;
 
 	Future<RangeResult> nextKeyValues(const int rowLimit, const int byteLimit) override { throw not_implemented(); }
 
-	// Returns the next chunk of serialized checkpoint.
 	Future<Standalone<StringRef>> nextChunk(const int byteLimit) override;
 
 	Future<Void> close() override;
 
 private:
-	ACTOR static Future<Void> doInit(RocksDBCheckpointReader* self) {
+	ACTOR static Future<Void> doInit(RocksDBCFCheckpointReader* self) {
 		ASSERT(self != nullptr);
 		try {
 			state Reference<IAsyncFile> _file = wait(IAsyncFileSystem::filesystem()->open(
@@ -62,7 +424,7 @@ private:
 		return Void();
 	}
 
-	ACTOR static Future<Standalone<StringRef>> getNextChunk(RocksDBCheckpointReader* self, int byteLimit) {
+	ACTOR static Future<Standalone<StringRef>> getNextChunk(RocksDBCFCheckpointReader* self, int byteLimit) {
 		int blockSize = std::min(64 * 1024, byteLimit); // Block size read from disk.
 		state Standalone<StringRef> buf = makeAlignedString(_PAGE_SIZE, blockSize);
 		int bytesRead = wait(self->file_->read(mutateString(buf), blockSize, self->offset_));
@@ -74,7 +436,7 @@ private:
 		return buf.substr(0, bytesRead);
 	}
 
-	ACTOR static Future<Void> doClose(RocksDBCheckpointReader* self) {
+	ACTOR static Future<Void> doClose(RocksDBCFCheckpointReader* self) {
 		wait(delay(0, TaskPriority::FetchKeys));
 		delete self;
 		return Void();
@@ -87,7 +449,7 @@ private:
 	std::string path_;
 };
 
-Future<Void> RocksDBCheckpointReader::init(StringRef token) {
+Future<Void> RocksDBCFCheckpointReader::init(StringRef token) {
 	ASSERT_EQ(this->checkpoint_.getFormat(), RocksDBColumnFamily);
 	const std::string name = token.toString();
 	this->offset_ = 0;
@@ -108,11 +470,11 @@ Future<Void> RocksDBCheckpointReader::init(StringRef token) {
 	return doInit(this);
 }
 
-Future<Standalone<StringRef>> RocksDBCheckpointReader::nextChunk(const int byteLimit) {
+Future<Standalone<StringRef>> RocksDBCFCheckpointReader::nextChunk(const int byteLimit) {
 	return getNextChunk(this, byteLimit);
 }
 
-Future<Void> RocksDBCheckpointReader::close() {
+Future<Void> RocksDBCFCheckpointReader::close() {
 	return doClose(this);
 }
 
@@ -216,13 +578,163 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
 	}
 }
 
+// TODO: Return when a file exceeds a limit.
+ACTOR Future<Void> fetchCheckpointRange(Database cx,
+                                        std::shared_ptr<CheckpointMetaData> metaData,
+                                        KeyRange range,
+                                        std::string dir,
+                                        std::shared_ptr<rocksdb::SstFileWriter> writer,
+                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun,
+                                        int maxRetries = 3) {
+	state std::string localFile = dir + "/" + metaData->checkpointID.toString() + ".sst";
+	RocksDBCheckpoint rcp = getRocksCheckpoint(*metaData);
+	TraceEvent("FetchCheckpointRange")
+	    .detail("InitialState", metaData->toString())
+	    .detail("RocksCheckpoint", rcp.toString());
+
+	for (const auto& file : rcp.fetchedFiles) {
+		ASSERT(!file.range.intersects(range));
+	}
+
+	state UID ssID = metaData->ssID;
+	state Transaction tr(cx);
+	state StorageServerInterface ssi;
+	loop {
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		try {
+			Optional<Value> ss = wait(tr.get(serverListKeyFor(ssID)));
+			if (!ss.present()) {
+				TraceEvent(SevWarnAlways, "FetchCheckpointRangeStorageServerNotFound")
+				    .detail("SSID", ssID)
+				    .detail("InitialState", metaData->toString());
+				throw checkpoint_not_found();
+			}
+			ssi = decodeServerListValue(ss.get());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+
+	ASSERT(ssi.id() == ssID);
+
+	state int attempt = 0;
+	state int64_t totalBytes = 0;
+	state rocksdb::Status status;
+	state Optional<Error> error;
+	loop {
+		totalBytes = 0;
+		++attempt;
+		try {
+			TraceEvent(SevInfo, "FetchCheckpointRangeBegin")
+			    .detail("CheckpointID", metaData->checkpointID)
+			    .detail("Range", range.toString())
+			    .detail("TargetStorageServerUID", ssID)
+			    .detail("LocalFile", localFile)
+			    .detail("Attempt", attempt)
+			    .log();
+
+			wait(IAsyncFileSystem::filesystem()->deleteFile(localFile, true));
+			status = writer->Open(localFile);
+			if (!status.ok()) {
+				Error e = statusToError(status);
+				TraceEvent(SevError, "FetchCheckpointRangeOpenFileError")
+				    .detail("LocalFile", localFile)
+				    .detail("Status", status.ToString());
+				throw e;
+			}
+
+			state ReplyPromiseStream<FetchCheckpointKeyValuesStreamReply> stream =
+			    ssi.fetchCheckpointKeyValues.getReplyStream(
+			        FetchCheckpointKeyValuesRequest(metaData->checkpointID, range));
+			TraceEvent(SevDebug, "FetchCheckpointKeyValuesReceivingData")
+			    .detail("CheckpointID", metaData->checkpointID)
+			    .detail("Range", range.toString())
+			    .detail("TargetStorageServerUID", ssID.toString())
+			    .detail("LocalFile", localFile)
+			    .detail("Attempt", attempt)
+			    .log();
+
+			loop {
+				FetchCheckpointKeyValuesStreamReply rep = waitNext(stream.getFuture());
+				for (int i = 0; i < rep.data.size(); ++i) {
+					status = writer->Put(toSlice(rep.data[i].key), toSlice(rep.data[i].value));
+					if (!status.ok()) {
+						Error e = statusToError(status);
+						TraceEvent(SevError, "FetchCheckpointRangeWriteError")
+						    .detail("LocalFile", localFile)
+						    .detail("Key", rep.data[i].key.toString())
+						    .detail("Value", rep.data[i].value.toString())
+						    .detail("Status", status.ToString());
+						throw e;
+					}
+					totalBytes += rep.data[i].expectedSize();
+				}
+			}
+		} catch (Error& e) {
+			Error err = e;
+			if (totalBytes > 0) {
+				status = writer->Finish();
+				if (!status.ok()) {
+					err = statusToError(status);
+				}
+			}
+			if (err.code() != error_code_end_of_stream) {
+				TraceEvent(SevWarn, "FetchCheckpointFileError")
+				    .errorUnsuppressed(err)
+				    .detail("CheckpointID", metaData->checkpointID)
+				    .detail("Range", range.toString())
+				    .detail("TargetStorageServerUID", ssID.toString())
+				    .detail("LocalFile", localFile)
+				    .detail("Attempt", attempt);
+				if (attempt >= maxRetries) {
+					error = err;
+					break;
+				}
+			} else {
+				if (totalBytes > 0) {
+					RocksDBCheckpoint rcp = getRocksCheckpoint(*metaData);
+					rcp.fetchedFiles.emplace_back(localFile, range, totalBytes);
+					rcp.checkpointDir = dir;
+					metaData->serializedCheckpoint = ObjectWriter::toValue(rcp, IncludeVersion());
+				}
+				if (!fileExists(localFile)) {
+					TraceEvent(SevWarn, "FetchCheckpointRangeEndFileNotFound")
+					    .detail("CheckpointID", metaData->checkpointID)
+					    .detail("Range", range.toString())
+					    .detail("TargetStorageServerUID", ssID.toString())
+					    .detail("LocalFile", localFile)
+					    .detail("Attempt", attempt)
+					    .detail("TotalBytes", totalBytes);
+				} else {
+					TraceEvent(SevInfo, "FetchCheckpointRangeEnd")
+					    .detail("CheckpointID", metaData->checkpointID)
+					    .detail("Range", range.toString())
+					    .detail("TargetStorageServerUID", ssID.toString())
+					    .detail("LocalFile", localFile)
+					    .detail("Attempt", attempt)
+					    .detail("TotalBytes", totalBytes);
+					break;
+				}
+			}
+		}
+	}
+
+	if (error.present()) {
+		throw error.get();
+	}
+
+	return Void();
+}
+
 } // namespace
 
 ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
                                                         CheckpointMetaData initialState,
                                                         std::string dir,
                                                         std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
-	TraceEvent("FetchRocksCheckpointBegin")
+	TraceEvent(SevInfo, "FetchRocksCheckpointBegin")
 	    .detail("InitialState", initialState.toString())
 	    .detail("CheckpointDir", dir);
 
@@ -230,50 +742,101 @@ ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
 
 	if (metaData->format == RocksDBColumnFamily) {
 		state RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(initialState);
-		TraceEvent("RocksDBCheckpointMetaData").detail("RocksCF", rocksCF.toString());
+		TraceEvent(SevDebug, "RocksDBCheckpointMetaData").detail("RocksCF", rocksCF.toString());
 
 		state int i = 0;
 		state std::vector<Future<Void>> fs;
 		for (; i < rocksCF.sstFiles.size(); ++i) {
 			fs.push_back(fetchCheckpointFile(cx, metaData, i, dir, cFun));
-			TraceEvent("GetCheckpointFetchingFile")
+			TraceEvent(SevDebug, "GetCheckpointFetchingFile")
 			    .detail("FileName", rocksCF.sstFiles[i].name)
 			    .detail("Server", metaData->ssID.toString());
 		}
 		wait(waitForAll(fs));
-	} else {
+	} else if (metaData->format == RocksDB) {
-		throw not_implemented();
+		std::shared_ptr<rocksdb::SstFileWriter> writer =
+		    std::make_shared<rocksdb::SstFileWriter>(rocksdb::EnvOptions(), rocksdb::Options());
+		wait(fetchCheckpointRange(cx, metaData, metaData->range, dir, writer, cFun));
 	}
 
 	return *metaData;
 }
 
-ACTOR Future<Void> deleteRocksCFCheckpoint(CheckpointMetaData checkpoint) {
+ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
-	ASSERT_EQ(checkpoint.getFormat(), RocksDBColumnFamily);
+	state CheckpointFormat format = checkpoint.getFormat();
-	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
-	TraceEvent("DeleteRocksColumnFamilyCheckpoint", checkpoint.checkpointID)
-	    .detail("CheckpointID", checkpoint.checkpointID)
-	    .detail("RocksCF", rocksCF.toString());
-
 	state std::unordered_set<std::string> dirs;
-	for (const LiveFileMetaData& file : rocksCF.sstFiles) {
+	if (format == RocksDBColumnFamily) {
-		dirs.insert(file.db_path);
+		RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
+		TraceEvent(SevInfo, "DeleteRocksColumnFamilyCheckpoint", checkpoint.checkpointID)
+		    .detail("CheckpointID", checkpoint.checkpointID)
+		    .detail("RocksCF", rocksCF.toString());
+
+		for (const LiveFileMetaData& file : rocksCF.sstFiles) {
+			dirs.insert(file.db_path);
+		}
+	} else if (format == RocksDB) {
+		RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
+		TraceEvent(SevInfo, "DeleteRocksCheckpoint", checkpoint.checkpointID)
+		    .detail("CheckpointID", checkpoint.checkpointID)
+		    .detail("RocksCheckpoint", rocksCheckpoint.toString());
+		dirs.insert(rocksCheckpoint.checkpointDir);
+	} else {
+		ASSERT(false);
 	}
 
 	state std::unordered_set<std::string>::iterator it = dirs.begin();
 	for (; it != dirs.end(); ++it) {
 		const std::string dir = *it;
 		platform::eraseDirectoryRecursive(dir);
-		TraceEvent("DeleteCheckpointRemovedDir", checkpoint.checkpointID)
+		TraceEvent(SevInfo, "DeleteCheckpointRemovedDir", checkpoint.checkpointID)
 		    .detail("CheckpointID", checkpoint.checkpointID)
 		    .detail("Dir", dir);
 		wait(delay(0, TaskPriority::FetchKeys));
 	}
+
 	return Void();
 }
+#else
+ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
+                                                        CheckpointMetaData initialState,
+                                                        std::string dir,
+                                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+	wait(delay(0));
+	return initialState;
+}
+
+ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
+	wait(delay(0));
+	return Void();
+}
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+
+int64_t getTotalFetchedBytes(const std::vector<CheckpointMetaData>& checkpoints) {
+	int64_t totalBytes = 0;
+	for (const auto& checkpoint : checkpoints) {
+		const CheckpointFormat format = checkpoint.getFormat();
+		if (format == RocksDBColumnFamily) {
+			// TODO: Returns the checkpoint size of a RocksDB Column Family.
+		} else if (format == RocksDB) {
+			auto rcp = getRocksCheckpoint(checkpoint);
+			for (const auto& file : rcp.fetchedFiles) {
+				totalBytes += file.size;
+			}
+		}
+	}
+	return totalBytes;
+}
 
 ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID) {
-	return new RocksDBCheckpointReader(checkpoint, logID);
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+	const CheckpointFormat format = checkpoint.getFormat();
+	if (format == RocksDBColumnFamily) {
+		return new RocksDBCFCheckpointReader(checkpoint, logID);
+	} else if (format == RocksDB) {
+		return new RocksDBCheckpointReader(checkpoint, logID);
+	}
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+	return nullptr;
 }
 
 RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint) {
@@ -281,4 +844,11 @@ RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint) {
 	ObjectReader reader(checkpoint.serializedCheckpoint.begin(), IncludeVersion());
 	reader.deserialize(rocksCF);
 	return rocksCF;
+}
+
+RocksDBCheckpoint getRocksCheckpoint(const CheckpointMetaData& checkpoint) {
+	RocksDBCheckpoint rocksCheckpoint;
+	ObjectReader reader(checkpoint.serializedCheckpoint.begin(), IncludeVersion());
+	reader.deserialize(rocksCheckpoint);
+	return rocksCheckpoint;
 }

fdbserver/RocksDBCheckpointUtils.actor.h

@@ -31,6 +31,26 @@
 
 #include "flow/actorcompiler.h" // has to be last include
 
+struct CheckpointFile {
+	constexpr static FileIdentifier file_identifier = 13804348;
+	std::string path;
+	KeyRange range;
+	int64_t size; // Logical bytes of the checkpoint.
+
+	CheckpointFile() = default;
+	CheckpointFile(std::string path, KeyRange range, int64_t size) : path(path), range(range), size(size) {}
+
+	std::string toString() const {
+		return "CheckpointFile:\nFile Name: " + this->path + "\nRange: " + range.toString() +
+		       "\nSize: " + std::to_string(size) + "\n";
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, path, range, size);
+	}
+};
+
 // Copied from rocksdb/metadata.h, so that we can add serializer.
 struct SstFileMetaData {
 	constexpr static FileIdentifier file_identifier = 3804347;
@@ -193,6 +213,34 @@ struct RocksDBColumnFamilyCheckpoint {
 	}
 };
 
+// Checkpoint metadata associated with RocksDB format.
+// The checkpoint is created via rocksdb::CreateCheckpoint().
+struct RocksDBCheckpoint {
+	constexpr static FileIdentifier file_identifier = 13804347;
+	std::string checkpointDir; // Checkpoint directory on the storage server.
+	std::vector<std::string> sstFiles; // All checkpoint files.
+	std::vector<CheckpointFile> fetchedFiles; // Used for fetchCheckpoint, to record the progress.
+
+	CheckpointFormat format() const { return RocksDB; }
+
+	std::string toString() const {
+		std::string res = "RocksDBCheckpoint:\nCheckpoint dir: " + checkpointDir + "\nFiles: ";
+		for (const std::string& file : sstFiles) {
+			res += (file + " ");
+		}
+		res += "\nFetched files:\n";
+		for (const auto& file : fetchedFiles) {
+			res += file.toString();
+		}
+		return res;
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, checkpointDir, sstFiles, fetchedFiles);
+	}
+};
+
 // Fetch the checkpoint file(s) to local dir, the checkpoint is specified by initialState.
 // If cFun is provided, the fetch progress can be checkpointed, so that next time, the fetch process
 // can be continued, in case of crash.
@@ -201,9 +249,15 @@ ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
                                                         std::string dir,
                                                         std::function<Future<Void>(const CheckpointMetaData&)> cFun);
 
-ACTOR Future<Void> deleteRocksCFCheckpoint(CheckpointMetaData checkpoint);
+// Returns the total logical bytes of all *fetched* checkpoints.
+int64_t getTotalFetchedBytes(const std::vector<CheckpointMetaData>& checkpoints);
+
+// Clean up on-disk files associated with checkpoint.
+ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint);
 
 ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID);
 
 RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint);
+
+RocksDBCheckpoint getRocksCheckpoint(const CheckpointMetaData& checkpoint);
 #endif

fdbserver/ServerCheckpoint.actor.cpp

@@ -24,12 +24,11 @@
 #include "flow/actorcompiler.h" // has to be last include
 
 ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint, UID logID) {
-	if (checkpoint.getFormat() == RocksDBColumnFamily) {
+	const CheckpointFormat format = checkpoint.getFormat();
+	if (format == RocksDBColumnFamily || format == RocksDB) {
 		return newRocksDBCheckpointReader(checkpoint, logID);
-	} else if (checkpoint.getFormat() == RocksDB) {
-		throw not_implemented();
 	} else {
-		ASSERT(false);
+		throw not_implemented();
 	}
 
 	return nullptr;
@@ -37,13 +36,11 @@ ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint, UID
 
 ACTOR Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint) {
 	wait(delay(0, TaskPriority::FetchKeys));
-
+	state CheckpointFormat format = checkpoint.getFormat();
-	if (checkpoint.getFormat() == RocksDBColumnFamily) {
+	if (format == RocksDBColumnFamily || format == RocksDB) {
-		wait(deleteRocksCFCheckpoint(checkpoint));
+		wait(deleteRocksCheckpoint(checkpoint));
-	} else if (checkpoint.getFormat() == RocksDB) {
-		throw not_implemented();
 	} else {
-		ASSERT(false);
+		throw not_implemented();
 	}
 
 	return Void();
@@ -53,15 +50,29 @@ ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
                                                  CheckpointMetaData initialState,
                                                  std::string dir,
                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+	TraceEvent("FetchCheckpointBegin", initialState.checkpointID).detail("CheckpointMetaData", initialState.toString());
 	state CheckpointMetaData result;
-	if (initialState.getFormat() == RocksDBColumnFamily) {
+	const CheckpointFormat format = initialState.getFormat();
+	if (format == RocksDBColumnFamily || format == RocksDB) {
 		CheckpointMetaData _result = wait(fetchRocksDBCheckpoint(cx, initialState, dir, cFun));
 		result = _result;
-	} else if (initialState.getFormat() == RocksDB) {
-		throw not_implemented();
 	} else {
-		ASSERT(false);
+		throw not_implemented();
 	}
 
+	TraceEvent("FetchCheckpointEnd", initialState.checkpointID).detail("CheckpointMetaData", result.toString());
 	return result;
+}
+
+ACTOR Future<std::vector<CheckpointMetaData>> fetchCheckpoints(
+    Database cx,
+    std::vector<CheckpointMetaData> initialStates,
+    std::string dir,
+    std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+	std::vector<Future<CheckpointMetaData>> actors;
+	for (const auto& checkpoint : initialStates) {
+		actors.push_back(fetchCheckpoint(cx, checkpoint, dir, cFun));
+	}
+	std::vector<CheckpointMetaData> res = wait(getAll(actors));
+	return res;
 }

fdbserver/ServerCheckpoint.actor.h

@@ -63,4 +63,10 @@ ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
                                                  CheckpointMetaData initialState,
                                                  std::string dir,
                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
+
+ACTOR Future<std::vector<CheckpointMetaData>> fetchCheckpoints(
+    Database cx,
+    std::vector<CheckpointMetaData> initialStates,
+    std::string dir,
+    std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
 #endif

fdbserver/SimKmsConnector.actor.cpp

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/SimKmsConnector.actor.h"
+#include "fdbserver/SimKmsConnector.h"
 
 #include "fdbrpc/sim_validation.h"
 #include "fdbserver/Knobs.h"
@@ -29,6 +29,7 @@
 #include "flow/FastRef.h"
 #include "flow/IRandom.h"
 #include "flow/ITrace.h"
+#include "flow/Trace.h"
 #include "flow/network.h"
 #include "flow/UnitTest.h"
 
@@ -79,6 +80,14 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
 			when(KmsConnLookupEKsByKeyIdsReq req = waitNext(interf.ekLookupByIds.getFuture())) {
 				state KmsConnLookupEKsByKeyIdsReq keysByIdsReq = req;
 				state KmsConnLookupEKsByKeyIdsRep keysByIdsRep;
+				state Optional<TraceEvent> dbgKIdTrace = keysByIdsReq.debugId.present()
+				                                             ? TraceEvent("SimKmsGetByKeyIds", interf.id())
+				                                             : Optional<TraceEvent>();
+
+				if (dbgKIdTrace.present()) {
+					dbgKIdTrace.get().setMaxEventLength(100000);
+					dbgKIdTrace.get().detail("DbgId", keysByIdsReq.debugId.get());
+				}
 
 				// Lookup corresponding EncryptKeyCtx for input keyId
 				for (const auto& item : req.encryptKeyIds) {
@@ -89,6 +98,12 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
 						    itr->first,
 						    StringRef(keysByIdsRep.arena, itr->second.get()->key),
 						    keysByIdsRep.arena);
+
+						if (dbgKIdTrace.present()) {
+							// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
+							dbgKIdTrace.get().detail(
+							    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.second, itr->first), "");
+						}
 					} else {
 						success = false;
 						break;
@@ -102,16 +117,29 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
 			when(KmsConnLookupEKsByDomainIdsReq req = waitNext(interf.ekLookupByDomainIds.getFuture())) {
 				state KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq = req;
 				state KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep;
+				state Optional<TraceEvent> dbgDIdTrace = keysByDomainIdReq.debugId.present()
+				                                             ? TraceEvent("SimKmsGetsByDomIds", interf.id())
+				                                             : Optional<TraceEvent>();
 
-				// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This would
+				if (dbgDIdTrace.present()) {
-				// mean multiple domains gets mapped to the same encryption key which is fine, the EncryptKeyStore
+					dbgDIdTrace.get().detail("DbgId", keysByDomainIdReq.debugId.get());
-				// guarantees that keyId -> plaintext encryptKey mapping is idempotent.
+				}
+
+				// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This
+				// would mean multiple domains gets mapped to the same encryption key which is fine, the
+				// EncryptKeyStore guarantees that keyId -> plaintext encryptKey mapping is idempotent.
 				for (EncryptCipherDomainId domainId : req.encryptDomainIds) {
 					EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
 					const auto& itr = ctx->simEncryptKeyStore.find(keyId);
 					if (itr != ctx->simEncryptKeyStore.end()) {
 						keysByDomainIdRep.cipherKeyDetails.emplace_back(
 						    domainId, keyId, StringRef(itr->second.get()->key), keysByDomainIdRep.arena);
+
+						if (dbgDIdTrace.present()) {
+							// {encryptId, baseCipherId} forms a unique tuple across encryption domains
+							dbgDIdTrace.get().detail(
+							    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), "");
+						}
 					} else {
 						success = false;
 						break;