Merge remote-tracking branch 'origin/main' into mako-cpp-async

bindings/c/CMakeLists.txt

@@ -123,6 +123,7 @@ if(NOT WIN32)
     test/apitester/TesterApiWrapper.h
     test/apitester/TesterTestSpec.cpp
     test/apitester/TesterTestSpec.h
+    test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
     test/apitester/TesterCancelTransactionWorkload.cpp
     test/apitester/TesterCorrectnessWorkload.cpp
     test/apitester/TesterKeyValueStore.cpp
@@ -272,7 +273,61 @@ endif()
             ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
             --test-dir
             ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
+            --tmp-dir
+            @TMP_DIR@
             )
+
+  add_fdbclient_test(
+    NAME fdb_c_api_tests_blob_granule
+    DISABLE_LOG_DUMP
+    API_TEST_BLOB_GRANULES_ENABLED
+    COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+            --cluster-file
+            @CLUSTER_FILE@
+            --tester-binary
+            $<TARGET_FILE:fdb_c_api_tester>
+            --external-client-library
+            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+            --test-dir
+            ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/blobgranuletests
+            --blob-granule-local-file-path
+            @DATA_DIR@/fdbblob/
+            )
+
+  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
+    add_test(NAME fdb_c_upgrade_single_threaded_630api
+      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
+            --upgrade-path "6.3.23" "7.0.0" "7.2.0"
+            --process-number 1
+            )
+
+    add_test(NAME fdb_c_upgrade_single_threaded_700api
+      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
+            --upgrade-path "7.0.0" "7.2.0"
+            --process-number 1
+            )
+
+    add_test(NAME fdb_c_upgrade_multi_threaded_630api
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+            --upgrade-path "6.3.23" "7.0.0" "7.2.0"
+            --process-number 3
+          )
+
+    add_test(NAME fdb_c_upgrade_multi_threaded_700api
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+            --upgrade-path "7.0.0" "7.2.0"
+            --process-number 3
+          )
+  endif()
+
 endif()
 
 set(c_workloads_srcs
@@ -316,6 +371,7 @@ fdb_install(
   FILES foundationdb/fdb_c.h
   ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
   ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
+  ${CMAKE_SOURCE_DIR}/bindings/c/foundationdb/fdb_c_types.h
   DESTINATION include
   DESTINATION_SUFFIX /foundationdb
   COMPONENT clients)

bindings/c/test/apitester/TesterApiWorkload.cpp

@@ -32,7 +32,33 @@ ApiWorkload::ApiWorkload(const WorkloadConfig& config) : WorkloadBase(config) {
 	maxKeysPerTransaction = config.getIntOption("maxKeysPerTransaction", 50);
 	initialSize = config.getIntOption("initialSize", 1000);
 	readExistingKeysRatio = config.getFloatOption("readExistingKeysRatio", 0.9);
+	runUntilStop = config.getBoolOption("runUntilStop", false);
+	numRandomOperations = config.getIntOption("numRandomOperations", 1000);
+	numOperationsForProgressCheck = config.getIntOption("numOperationsForProgressCheck", 10);
 	keyPrefix = fmt::format("{}/", workloadId);
+	numRandomOpLeft = 0;
+	stopReceived = false;
+	checkingProgress = false;
+	apiVersion = config.apiVersion;
+}
+
+IWorkloadControlIfc* ApiWorkload::getControlIfc() {
+	if (runUntilStop) {
+		return this;
+	} else {
+		return nullptr;
+	}
+}
+
+void ApiWorkload::stop() {
+	ASSERT(runUntilStop);
+	stopReceived = true;
+}
+
+void ApiWorkload::checkProgress() {
+	ASSERT(runUntilStop);
+	numRandomOpLeft = numOperationsForProgressCheck;
+	checkingProgress = true;
 }
 
 void ApiWorkload::start() {
@@ -48,6 +74,37 @@ void ApiWorkload::start() {
 	});
 }
 
+void ApiWorkload::runTests() {
+	if (!runUntilStop) {
+		numRandomOpLeft = numRandomOperations;
+	}
+	randomOperations();
+}
+
+void ApiWorkload::randomOperations() {
+	if (runUntilStop) {
+		if (stopReceived)
+			return;
+		if (checkingProgress) {
+			int numLeft = numRandomOpLeft--;
+			if (numLeft == 0) {
+				checkingProgress = false;
+				confirmProgress();
+			}
+		}
+	} else {
+		int numLeft = numRandomOpLeft--;
+		if (numLeft == 0)
+			return;
+	}
+	randomOperation([this]() { randomOperations(); });
+}
+
+void ApiWorkload::randomOperation(TTaskFct cont) {
+	// Must be overridden if used
+	ASSERT(false);
+}
+
 std::string ApiWorkload::randomKeyName() {
 	return keyPrefix + Random::get().randomStringLowerCase(minKeyLength, maxKeyLength);
 }
@@ -126,4 +183,63 @@ void ApiWorkload::populateData(TTaskFct cont) {
 	}
 }
 
+void ApiWorkload::randomInsertOp(TTaskFct cont) {
+	int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
+	auto kvPairs = std::make_shared<std::vector<KeyValue>>();
+	for (int i = 0; i < numKeys; i++) {
+		kvPairs->push_back(KeyValue{ randomNotExistingKey(), randomValue() });
+	}
+	execTransaction(
+	    [kvPairs](auto ctx) {
+		    for (const KeyValue& kv : *kvPairs) {
+			    ctx->tx()->set(kv.key, kv.value);
+		    }
+		    ctx->commit();
+	    },
+	    [this, kvPairs, cont]() {
+		    for (const KeyValue& kv : *kvPairs) {
+			    store.set(kv.key, kv.value);
+		    }
+		    schedule(cont);
+	    });
+}
+
+void ApiWorkload::randomClearOp(TTaskFct cont) {
+	int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
+	auto keys = std::make_shared<std::vector<std::string>>();
+	for (int i = 0; i < numKeys; i++) {
+		keys->push_back(randomExistingKey());
+	}
+	execTransaction(
+	    [keys](auto ctx) {
+		    for (const auto& key : *keys) {
+			    ctx->tx()->clear(key);
+		    }
+		    ctx->commit();
+	    },
+	    [this, keys, cont]() {
+		    for (const auto& key : *keys) {
+			    store.clear(key);
+		    }
+		    schedule(cont);
+	    });
+}
+
+void ApiWorkload::randomClearRangeOp(TTaskFct cont) {
+	std::string begin = randomKeyName();
+	std::string end = randomKeyName();
+	if (begin > end) {
+		std::swap(begin, end);
+	}
+	execTransaction(
+	    [begin, end](auto ctx) {
+		    ctx->tx()->clearRange(begin, end);
+		    ctx->commit();
+	    },
+	    [this, begin, end, cont]() {
+		    store.clear(begin, end);
+		    schedule(cont);
+	    });
+}
+
 } // namespace FdbApiTester

bindings/c/test/apitester/TesterApiWorkload.h

@@ -23,6 +23,7 @@
 
 #include "TesterWorkload.h"
 #include "TesterKeyValueStore.h"
+#include <atomic>
 
 namespace FdbApiTester {
 
@@ -30,14 +31,28 @@ namespace FdbApiTester {
  * Base class for implementing API testing workloads.
  * Provides various helper methods and reusable configuration parameters
  */
-class ApiWorkload : public WorkloadBase {
+class ApiWorkload : public WorkloadBase, IWorkloadControlIfc {
 public:
 	void start() override;
 
-	// Method to be overridden to run specific tests
-	virtual void runTests() = 0;
+	IWorkloadControlIfc* getControlIfc() override;
+
+	virtual void stop() override;
+
+	virtual void checkProgress() override;
+
+	// Running specific tests
+	// The default implementation generates a workload consisting of
+	// random operations generated by randomOperation
+	virtual void runTests();
+
+	// Generate a random operation and schedule the continuation when done
+	virtual void randomOperation(TTaskFct cont);
 
 protected:
+	// Selected FDB API version
+	int apiVersion;
+
 	// The minimum length of a key
 	int minKeyLength;
 
@@ -59,6 +74,25 @@ protected:
 	// The ratio of reading existing keys
 	double readExistingKeysRatio;
 
+	// Run the workload until explicit stop
+	bool runUntilStop;
+
+	// The number of operations to be executed (for runUntilStop=false)
+	int numRandomOperations;
+
+	// The number of transactions to be completed for
+	// a successful test progress check
+	int numOperationsForProgressCheck;
+
+	// Stop command received (for runUntilStop=true)
+	std::atomic<bool> stopReceived;
+
+	// Progress check is active (for runUntilStop=true)
+	std::atomic<bool> checkingProgress;
+
+	// Number of random operations left (for runUntilStop=false)
+	std::atomic<int> numRandomOpLeft;
+
 	// Key prefix
 	std::string keyPrefix;
 
@@ -80,8 +114,15 @@ protected:
 	// Clear the data of the workload
 	void clearData(TTaskFct cont);
 
+	// common operations
+	void randomInsertOp(TTaskFct cont);
+	void randomClearOp(TTaskFct cont);
+	void randomClearRangeOp(TTaskFct cont);
+
 private:
 	void populateDataTx(TTaskFct cont);
+
+	void randomOperations();
 };
 
 } // namespace FdbApiTester

bindings/c/test/apitester/TesterApiWrapper.cpp

@@ -18,9 +18,9 @@
  * limitations under the License.
  */
 #include "TesterApiWrapper.h"
-#include "TesterUtil.h"
 #include <cstdint>
 #include <fmt/format.h>
+#include <fstream>
 
 namespace FdbApiTester {
 
@@ -28,7 +28,7 @@ namespace {
 
 void fdb_check(fdb_error_t e) {
 	if (e) {
-		fmt::print(stderr, "Unexpected error: %s\n", fdb_get_error(e));
+		fmt::print(stderr, "Unexpected error: {}\n", fdb_get_error(e));
 		std::abort();
 	}
 }
@@ -60,6 +60,56 @@ std::optional<std::string> ValueFuture::getValue() const {
 	return out_present ? std::make_optional(std::string((const char*)val, vallen)) : std::nullopt;
 }
 
+std::vector<KeyValue> KeyRangesFuture::getKeyRanges() const {
+	ASSERT(future_);
+
+	int count;
+	const FDBKeyRange* ranges;
+
+	fdb_check(fdb_future_get_keyrange_array(future_.get(), &ranges, &count));
+	std::vector<KeyValue> result;
+	result.reserve(count);
+	for (int i = 0; i < count; i++) {
+		FDBKeyRange kr = *ranges++;
+		KeyValue rkv;
+		rkv.key = std::string((const char*)kr.begin_key, kr.begin_key_length);
+		rkv.value = std::string((const char*)kr.end_key, kr.end_key_length);
+		result.push_back(rkv);
+	}
+
+	return result;
+}
+
+Result::Result(FDBResult* r) : result_(r, fdb_result_destroy) {}
+
+std::vector<KeyValue> KeyValuesResult::getKeyValues(bool* more_out) {
+	ASSERT(result_);
+
+	int count;
+	const FDBKeyValue* kvs;
+	int more;
+
+	std::vector<KeyValue> result;
+
+	error_ = fdb_result_get_keyvalue_array(result_.get(), &kvs, &count, &more);
+
+	if (error_ != error_code_success) {
+		return result;
+	}
+
+	result.reserve(count);
+	for (int i = 0; i < count; i++) {
+		FDBKeyValue kv = *kvs++;
+		KeyValue rkv;
+		rkv.key = std::string((const char*)kv.key, kv.key_length);
+		rkv.value = std::string((const char*)kv.value, kv.value_length);
+		result.push_back(rkv);
+	}
+	*more_out = more;
+
+	return result;
+}
+
 // Given an FDBDatabase, initializes a new transaction.
 Transaction::Transaction(FDBTransaction* tx) : tx_(tx, fdb_transaction_destroy) {}
 
@@ -109,6 +159,87 @@ fdb_error_t Transaction::setOption(FDBTransactionOption option) {
 	return fdb_transaction_set_option(tx_.get(), option, reinterpret_cast<const uint8_t*>(""), 0);
 }
 
+class TesterGranuleContext {
+public:
+	std::unordered_map<int64_t, uint8_t*> loadsInProgress;
+	int64_t nextId = 0;
+	std::string basePath;
+
+	~TesterGranuleContext() {
+		// if there was an error or not all loads finished, delete data
+		for (auto& it : loadsInProgress) {
+			uint8_t* dataToFree = it.second;
+			delete dataToFree;
+		}
+	}
+};
+
+static int64_t granule_start_load(const char* filename,
+                                  int filenameLength,
+                                  int64_t offset,
+                                  int64_t length,
+                                  int64_t fullFileLength,
+                                  void* context) {
+
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	int64_t loadId = ctx->nextId++;
+
+	uint8_t* buffer = new uint8_t[length];
+	std::ifstream fin(ctx->basePath + std::string(filename, filenameLength), std::ios::in | std::ios::binary);
+	fin.seekg(offset);
+	fin.read((char*)buffer, length);
+
+	ctx->loadsInProgress.insert({ loadId, buffer });
+
+	return loadId;
+}
+
+static uint8_t* granule_get_load(int64_t loadId, void* context) {
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	return ctx->loadsInProgress.at(loadId);
+}
+
+static void granule_free_load(int64_t loadId, void* context) {
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	auto it = ctx->loadsInProgress.find(loadId);
+	uint8_t* dataToFree = it->second;
+	delete dataToFree;
+
+	ctx->loadsInProgress.erase(it);
+}
+
+KeyValuesResult Transaction::readBlobGranules(std::string_view begin,
+                                              std::string_view end,
+                                              const std::string& basePath) {
+	ASSERT(tx_);
+
+	TesterGranuleContext testerContext;
+	testerContext.basePath = basePath;
+
+	FDBReadBlobGranuleContext granuleContext;
+	granuleContext.userContext = &testerContext;
+	granuleContext.debugNoMaterialize = false;
+	granuleContext.granuleParallelism = 1;
+	granuleContext.start_load_f = &granule_start_load;
+	granuleContext.get_load_f = &granule_get_load;
+	granuleContext.free_load_f = &granule_free_load;
+
+	return KeyValuesResult(fdb_transaction_read_blob_granules(tx_.get(),
+	                                                          (const uint8_t*)begin.data(),
+	                                                          begin.size(),
+	                                                          (const uint8_t*)end.data(),
+	                                                          end.size(),
+	                                                          0 /* beginVersion */,
+	                                                          -2 /* latest read version */,
+	                                                          granuleContext));
+}
+
+KeyRangesFuture Transaction::getBlobGranuleRanges(std::string_view begin, std::string_view end) {
+	ASSERT(tx_);
+	return KeyRangesFuture(fdb_transaction_get_blob_granule_ranges(
+	    tx_.get(), (const uint8_t*)begin.data(), begin.size(), (const uint8_t*)end.data(), end.size()));
+}
+
 fdb_error_t FdbApi::setOption(FDBNetworkOption option, std::string_view value) {
 	return fdb_network_set_option(option, reinterpret_cast<const uint8_t*>(value.data()), value.size());
 }

bindings/c/test/apitester/TesterApiWrapper.h

@@ -26,6 +26,7 @@
 #include <string_view>
 #include <optional>
 #include <memory>
+#include <unordered_map>
 
 #define FDB_API_VERSION 720
 #include "bindings/c/foundationdb/fdb_c.h"
@@ -35,6 +36,8 @@
 
 #include "flow/error_definitions.h"
 
+#include "TesterUtil.h"
+
 namespace FdbApiTester {
 
 // Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
@@ -62,6 +65,37 @@ public:
 	std::optional<std::string> getValue() const;
 };
 
+class KeyRangesFuture : public Future {
+public:
+	KeyRangesFuture() = default;
+	KeyRangesFuture(FDBFuture* f) : Future(f) {}
+	std::vector<KeyValue> getKeyRanges() const;
+};
+
+class Result {
+public:
+	Result() = default;
+	Result(FDBResult* r);
+
+	FDBResult* fdbResult() { return result_.get(); };
+
+	fdb_error_t getError() const { return error_; }
+
+	explicit operator bool() const { return result_ != nullptr; };
+
+	fdb_error_t error_ = error_code_client_invalid_operation; // have to call getX function to set this
+
+protected:
+	std::shared_ptr<FDBResult> result_;
+};
+
+class KeyValuesResult : public Result {
+public:
+	KeyValuesResult() = default;
+	KeyValuesResult(FDBResult* f) : Result(f) {}
+	std::vector<KeyValue> getKeyValues(bool* more_out);
+};
+
 class Transaction {
 public:
 	Transaction() = default;
@@ -76,6 +110,9 @@ public:
 	void reset();
 	fdb_error_t setOption(FDBTransactionOption option);
 
+	KeyValuesResult readBlobGranules(std::string_view begin, std::string_view end, const std::string& basePath);
+	KeyRangesFuture getBlobGranuleRanges(std::string_view begin, std::string_view end);
+
 private:
 	std::shared_ptr<FDBTransaction> tx_;
 };

bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp

@@ -0,0 +1,159 @@
+/*
+ * TesterBlobGranuleCorrectnessWorkload.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "TesterApiWorkload.h"
+#include "TesterUtil.h"
+#include <memory>
+#include <fmt/format.h>
+
+namespace FdbApiTester {
+
+class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
+public:
+	ApiBlobGranuleCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {
+		// sometimes don't do range clears
+		if (Random::get().randomInt(0, 1) == 0) {
+			excludedOpTypes.push_back(OP_CLEAR_RANGE);
+		}
+	}
+
+private:
+	enum OpType { OP_INSERT, OP_CLEAR, OP_CLEAR_RANGE, OP_READ, OP_GET_RANGES, OP_LAST = OP_GET_RANGES };
+	std::vector<OpType> excludedOpTypes;
+
+	void randomReadOp(TTaskFct cont) {
+		std::string begin = randomKeyName();
+		std::string end = randomKeyName();
+		auto results = std::make_shared<std::vector<KeyValue>>();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execTransaction(
+		    [begin, end, results](auto ctx) {
+			    ctx->tx()->setOption(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE);
+			    KeyValuesResult res = ctx->tx()->readBlobGranules(begin, end, ctx->getBGBasePath());
+			    bool more;
+			    (*results) = res.getKeyValues(&more);
+			    ASSERT(!more);
+			    if (res.getError() != error_code_success) {
+				    ctx->onError(res.getError());
+			    } else {
+				    ctx->done();
+			    }
+		    },
+		    [this, begin, end, results, cont]() {
+			    std::vector<KeyValue> expected = store.getRange(begin, end, store.size(), false);
+			    if (results->size() != expected.size()) {
+				    error(fmt::format("randomReadOp result size mismatch. expected: {} actual: {}",
+				                      expected.size(),
+				                      results->size()));
+			    }
+			    ASSERT(results->size() == expected.size());
+
+			    for (int i = 0; i < results->size(); i++) {
+				    if ((*results)[i].key != expected[i].key) {
+					    error(fmt::format("randomReadOp key mismatch at {}/{}. expected: {} actual: {}",
+					                      i,
+					                      results->size(),
+					                      expected[i].key,
+					                      (*results)[i].key));
+				    }
+				    ASSERT((*results)[i].key == expected[i].key);
+
+				    if ((*results)[i].value != expected[i].value) {
+					    error(
+					        fmt::format("randomReadOp value mismatch at {}/{}. key: {} expected: {:.80} actual: {:.80}",
+					                    i,
+					                    results->size(),
+					                    expected[i].key,
+					                    expected[i].value,
+					                    (*results)[i].value));
+				    }
+				    ASSERT((*results)[i].value == expected[i].value);
+			    }
+			    schedule(cont);
+		    });
+	}
+
+	void randomGetRangesOp(TTaskFct cont) {
+		std::string begin = randomKeyName();
+		std::string end = randomKeyName();
+		auto results = std::make_shared<std::vector<KeyValue>>();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execTransaction(
+		    [begin, end, results](auto ctx) {
+			    KeyRangesFuture f = ctx->tx()->getBlobGranuleRanges(begin, end);
+			    ctx->continueAfter(
+			        f,
+			        [ctx, f, results]() {
+				        (*results) = f.getKeyRanges();
+				        ctx->done();
+			        },
+			        true);
+		    },
+		    [this, begin, end, results, cont]() {
+			    ASSERT(results->size() > 0);
+			    ASSERT(results->front().key <= begin);
+			    ASSERT(results->back().value >= end);
+
+			    for (int i = 0; i < results->size(); i++) {
+				    // no empty or inverted ranges
+				    ASSERT((*results)[i].key < (*results)[i].value);
+			    }
+
+			    for (int i = 1; i < results->size(); i++) {
+				    // ranges contain entire requested key range
+				    ASSERT((*results)[i].key == (*results)[i - 1].value);
+			    }
+
+			    schedule(cont);
+		    });
+	}
+
+	void randomOperation(TTaskFct cont) {
+		OpType txType = (store.size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
+		while (std::count(excludedOpTypes.begin(), excludedOpTypes.end(), txType)) {
+			txType = (OpType)Random::get().randomInt(0, OP_LAST);
+		}
+		switch (txType) {
+		case OP_INSERT:
+			randomInsertOp(cont);
+			break;
+		case OP_CLEAR:
+			randomClearOp(cont);
+			break;
+		case OP_CLEAR_RANGE:
+			randomClearRangeOp(cont);
+			break;
+		case OP_READ:
+			randomReadOp(cont);
+			break;
+		case OP_GET_RANGES:
+			randomGetRangesOp(cont);
+			break;
+		}
+	}
+};
+
+WorkloadFactory<ApiBlobGranuleCorrectnessWorkload> ApiBlobGranuleCorrectnessWorkloadFactory(
+    "ApiBlobGranuleCorrectness");
+
+} // namespace FdbApiTester

bindings/c/test/apitester/TesterCancelTransactionWorkload.cpp

@@ -24,22 +24,11 @@ namespace FdbApiTester {
 
 class CancelTransactionWorkload : public ApiWorkload {
 public:
-	CancelTransactionWorkload(const WorkloadConfig& config) : ApiWorkload(config) {
-		numRandomOperations = config.getIntOption("numRandomOperations", 1000);
-		numOpLeft = numRandomOperations;
-	}
-
-	void runTests() override { randomOperations(); }
+	CancelTransactionWorkload(const WorkloadConfig& config) : ApiWorkload(config) {}
 
 private:
 	enum OpType { OP_CANCEL_GET, OP_CANCEL_AFTER_FIRST_GET, OP_LAST = OP_CANCEL_AFTER_FIRST_GET };
 
-	// The number of operations to be executed
-	int numRandomOperations;
-
-	// Operations counter
-	int numOpLeft;
-
 	// Start multiple concurrent gets and cancel the transaction
 	void randomCancelGetTx(TTaskFct cont) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
@@ -87,7 +76,7 @@ private:
 		    [this, cont]() { schedule(cont); });
 	}
 
-	void randomOperation(TTaskFct cont) {
+	void randomOperation(TTaskFct cont) override {
 		OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
 		switch (txType) {
 		case OP_CANCEL_GET:
@@ -98,14 +87,6 @@ private:
 			break;
 		}
 	}
-
-	void randomOperations() {
-		if (numOpLeft == 0)
-			return;
-
-		numOpLeft--;
-		randomOperation([this]() { randomOperations(); });
-	}
 };
 
 WorkloadFactory<CancelTransactionWorkload> MiscTestWorkloadFactory("CancelTransaction");

bindings/c/test/apitester/TesterCorrectnessWorkload.cpp

@@ -26,43 +26,11 @@ namespace FdbApiTester {
 
 class ApiCorrectnessWorkload : public ApiWorkload {
 public:
-	ApiCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {
-		numRandomOperations = config.getIntOption("numRandomOperations", 1000);
-		numOpLeft = numRandomOperations;
-	}
-
-	void runTests() override { randomOperations(); }
+	ApiCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {}
 
 private:
 	enum OpType { OP_INSERT, OP_GET, OP_CLEAR, OP_CLEAR_RANGE, OP_COMMIT_READ, OP_LAST = OP_COMMIT_READ };
 
-	// The number of operations to be executed
-	int numRandomOperations;
-
-	// Operations counter
-	int numOpLeft;
-
-	void randomInsertOp(TTaskFct cont) {
-		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
-		auto kvPairs = std::make_shared<std::vector<KeyValue>>();
-		for (int i = 0; i < numKeys; i++) {
-			kvPairs->push_back(KeyValue{ randomNotExistingKey(), randomValue() });
-		}
-		execTransaction(
-		    [kvPairs](auto ctx) {
-			    for (const KeyValue& kv : *kvPairs) {
-				    ctx->tx()->set(kv.key, kv.value);
-			    }
-			    ctx->commit();
-		    },
-		    [this, kvPairs, cont]() {
-			    for (const KeyValue& kv : *kvPairs) {
-				    store.set(kv.key, kv.value);
-			    }
-			    schedule(cont);
-		    });
-	}
-
 	void randomCommitReadOp(TTaskFct cont) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto kvPairs = std::make_shared<std::vector<KeyValue>>();
@@ -82,8 +50,11 @@ private:
 			    }
 			    auto results = std::make_shared<std::vector<std::optional<std::string>>>();
 			    execTransaction(
-			        [kvPairs, results](auto ctx) {
-				        ctx->tx()->setOption(FDB_TR_OPTION_USE_GRV_CACHE);
+			        [kvPairs, results, this](auto ctx) {
+				        if (apiVersion >= 710) {
+					        // Test GRV caching in 7.1 and later
+					        ctx->tx()->setOption(FDB_TR_OPTION_USE_GRV_CACHE);
+				        }
 				        auto futures = std::make_shared<std::vector<Future>>();
 				        for (const auto& kv : *kvPairs) {
 					        futures->push_back(ctx->tx()->get(kv.key, false));
@@ -153,44 +124,6 @@ private:
 		    });
 	}
 
-	void randomClearOp(TTaskFct cont) {
-		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
-		auto keys = std::make_shared<std::vector<std::string>>();
-		for (int i = 0; i < numKeys; i++) {
-			keys->push_back(randomExistingKey());
-		}
-		execTransaction(
-		    [keys](auto ctx) {
-			    for (const auto& key : *keys) {
-				    ctx->tx()->clear(key);
-			    }
-			    ctx->commit();
-		    },
-		    [this, keys, cont]() {
-			    for (const auto& key : *keys) {
-				    store.clear(key);
-			    }
-			    schedule(cont);
-		    });
-	}
-
-	void randomClearRangeOp(TTaskFct cont) {
-		std::string begin = randomKeyName();
-		std::string end = randomKeyName();
-		if (begin > end) {
-			std::swap(begin, end);
-		}
-		execTransaction(
-		    [begin, end](auto ctx) {
-			    ctx->tx()->clearRange(begin, end);
-			    ctx->commit();
-		    },
-		    [this, begin, end, cont]() {
-			    store.clear(begin, end);
-			    schedule(cont);
-		    });
-	}
-
 	void randomOperation(TTaskFct cont) {
 		OpType txType = (store.size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
 		switch (txType) {
@@ -211,14 +144,6 @@ private:
 			break;
 		}
 	}
-
-	void randomOperations() {
-		if (numOpLeft == 0)
-			return;
-
-		numOpLeft--;
-		randomOperation([this]() { randomOperations(); });
-	}
 };
 
 WorkloadFactory<ApiCorrectnessWorkload> ApiCorrectnessWorkloadFactory("ApiCorrectness");

bindings/c/test/apitester/TesterKeyValueStore.h

@@ -30,12 +30,9 @@
 #include <vector>
 #include <mutex>
 
-namespace FdbApiTester {
+#include "TesterUtil.h"
 
-struct KeyValue {
-	std::string key;
-	std::string value;
-};
+namespace FdbApiTester {
 
 class KeyValueStore {
 public:

bindings/c/test/apitester/TesterOptions.h

@@ -29,19 +29,28 @@ namespace FdbApiTester {
 
 class TesterOptions {
 public:
+	// FDB API version, using the latest version by default
+	int apiVersion = FDB_API_VERSION;
 	std::string clusterFile;
 	bool trace = false;
 	std::string traceDir;
-	std::string traceFormat;
+	std::string traceFormat = "xml";
 	std::string logGroup;
 	std::string externalClientLibrary;
+	std::string externalClientDir;
+	std::string tmpDir;
+	bool disableLocalClient = false;
 	std::string testFile;
+	std::string inputPipeName;
+	std::string outputPipeName;
+	int transactionRetryLimit = 0;
 	int numFdbThreads;
 	int numClientThreads;
 	int numDatabases;
 	int numClients;
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
+	std::string bgBasePath;
 };
 
 } // namespace FdbApiTester

bindings/c/test/apitester/TesterTestSpec.cpp

@@ -45,10 +45,6 @@ std::unordered_map<std::string, std::function<void(const std::string& value, Tes
 	  [](const std::string& value, TestSpec* spec) { //
 	      spec->title = value;
 	  } },
-	{ "apiVersion",
-	  [](const std::string& value, TestSpec* spec) { //
-	      processIntOption(value, "apiVersion", spec->apiVersion, 700, 720);
-	  } },
 	{ "blockOnFutures",
 	  [](const std::string& value, TestSpec* spec) { //
 	      spec->blockOnFutures = (value == "true");

bindings/c/test/apitester/TesterTestSpec.h

@@ -42,9 +42,6 @@ struct TestSpec {
 	// Title of the test
 	std::string title;
 
-	// FDB API version, using the latest version by default
-	int apiVersion = FDB_API_VERSION;
-
 	// Use blocking waits on futures instead of scheduling callbacks
 	bool blockOnFutures = false;
 

bindings/c/test/apitester/TesterTransactionExecutor.cpp

@@ -20,8 +20,10 @@
 
 #include "TesterTransactionExecutor.h"
 #include "TesterUtil.h"
+#include "foundationdb/fdb_c_types.h"
 #include "test/apitester/TesterScheduler.h"
 #include <memory>
+#include <stdexcept>
 #include <unordered_map>
 #include <mutex>
 #include <atomic>
@@ -31,6 +33,9 @@
 
 namespace FdbApiTester {
 
+constexpr int LONG_WAIT_TIME_US = 1000000;
+constexpr int LARGE_NUMBER_OF_RETRIES = 5;
+
 void TransactionActorBase::complete(fdb_error_t err) {
 	error = err;
 	context = {};
@@ -69,8 +74,11 @@ public:
 	TransactionContextBase(FDBTransaction* tx,
 	                       std::shared_ptr<ITransactionActor> txActor,
 	                       TTaskFct cont,
-	                       IScheduler* scheduler)
-	  : fdbTx(tx), txActor(txActor), contAfterDone(cont), scheduler(scheduler), txState(TxState::IN_PROGRESS) {}
+	                       IScheduler* scheduler,
+	                       int retryLimit,
+	                       std::string bgBasePath)
+	  : fdbTx(tx), txActor(txActor), contAfterDone(cont), scheduler(scheduler), retryLimit(retryLimit),
+	    txState(TxState::IN_PROGRESS), commitCalled(false), bgBasePath(bgBasePath) {}
 
 	// A state machine:
 	// IN_PROGRESS -> (ON_ERROR -> IN_PROGRESS)* [-> ON_ERROR] -> DONE
@@ -87,6 +95,7 @@ public:
 		if (txState != TxState::IN_PROGRESS) {
 			return;
 		}
+		commitCalled = true;
 		lock.unlock();
 		Future f = fdbTx.commit();
 		auto thisRef = shared_from_this();
@@ -102,6 +111,11 @@ public:
 		}
 		txState = TxState::DONE;
 		lock.unlock();
+		if (retriedErrors.size() >= LARGE_NUMBER_OF_RETRIES) {
+			fmt::print("Transaction succeeded after {} retries on errors: {}\n",
+			           retriedErrors.size(),
+			           fmt::join(retriedErrors, ", "));
+		}
 		// cancel transaction so that any pending operations on it
 		// fail gracefully
 		fdbTx.cancel();
@@ -110,6 +124,8 @@ public:
 		contAfterDone();
 	}
 
+	std::string getBGBasePath() override { return bgBasePath; }
+
 protected:
 	virtual void doContinueAfter(Future f, TTaskFct cont, bool retryOnError) = 0;
 
@@ -146,11 +162,29 @@ protected:
 		} else {
 			std::unique_lock<std::mutex> lock(mutex);
 			txState = TxState::IN_PROGRESS;
+			commitCalled = false;
 			lock.unlock();
 			txActor->start();
 		}
 	}
 
+	// Checks if a transaction can be retried. Fails the transaction if the check fails
+	bool canRetry(fdb_error_t lastErr) {
+		ASSERT(txState == TxState::ON_ERROR);
+		retriedErrors.push_back(lastErr);
+		if (retryLimit == 0 || retriedErrors.size() <= retryLimit) {
+			if (retriedErrors.size() == LARGE_NUMBER_OF_RETRIES) {
+				fmt::print("Transaction already retried {} times, on errors: {}\n",
+				           retriedErrors.size(),
+				           fmt::join(retriedErrors, ", "));
+			}
+			return true;
+		}
+		fmt::print("Transaction retry limit reached. Retried on errors: {}\n", fmt::join(retriedErrors, ", "));
+		transactionFailed(lastErr);
+		return false;
+	}
+
 	// FDB transaction
 	Transaction fdbTx;
 
@@ -166,11 +200,29 @@ protected:
 	// Reference to the scheduler
 	IScheduler* scheduler;
 
+	// Retry limit
+	int retryLimit;
+
 	// Transaction execution state
 	TxState txState;
 
 	// onError future used in ON_ERROR state
 	Future onErrorFuture;
+
+	// The error code on which onError was called
+	fdb_error_t onErrorArg;
+
+	// The time point of calling onError
+	TimePoint onErrorCallTimePoint;
+
+	// Transaction is committed or being committed
+	bool commitCalled;
+
+	// A history of errors on which the transaction was retried
+	std::vector<fdb_error_t> retriedErrors;
+
+	// blob granule base path
+	std::string bgBasePath;
 };
 
 /**
@@ -181,8 +233,10 @@ public:
 	BlockingTransactionContext(FDBTransaction* tx,
 	                           std::shared_ptr<ITransactionActor> txActor,
 	                           TTaskFct cont,
-	                           IScheduler* scheduler)
-	  : TransactionContextBase(tx, txActor, cont, scheduler) {}
+	                           IScheduler* scheduler,
+	                           int retryLimit,
+	                           std::string bgBasePath)
+	  : TransactionContextBase(tx, txActor, cont, scheduler, retryLimit, bgBasePath) {}
 
 protected:
 	void doContinueAfter(Future f, TTaskFct cont, bool retryOnError) override {
@@ -197,12 +251,21 @@ protected:
 			return;
 		}
 		lock.unlock();
+		auto start = timeNow();
 		fdb_error_t err = fdb_future_block_until_ready(f.fdbFuture());
 		if (err) {
 			transactionFailed(err);
 			return;
 		}
 		err = f.getError();
+		auto waitTimeUs = timeElapsedInUs(start);
+		if (waitTimeUs > LONG_WAIT_TIME_US) {
+			fmt::print("Long waiting time on a future: {:.3f}s, return code {} ({}), commit called: {}\n",
+			           microsecToSec(waitTimeUs),
+			           err,
+			           fdb_get_error(err),
+			           commitCalled);
+		}
 		if (err == error_code_transaction_cancelled) {
 			return;
 		}
@@ -223,13 +286,29 @@ protected:
 		txState = TxState::ON_ERROR;
 		lock.unlock();
 
+		if (!canRetry(err)) {
+			return;
+		}
+
 		ASSERT(!onErrorFuture);
 		onErrorFuture = fdbTx.onError(err);
+		onErrorArg = err;
+
+		auto start = timeNow();
 		fdb_error_t err2 = fdb_future_block_until_ready(onErrorFuture.fdbFuture());
 		if (err2) {
 			transactionFailed(err2);
 			return;
 		}
+		auto waitTimeUs = timeElapsedInUs(start);
+		if (waitTimeUs > LONG_WAIT_TIME_US) {
+			fdb_error_t err3 = onErrorFuture.getError();
+			fmt::print("Long waiting time on onError({}) future: {:.3f}s, return code {} ({})\n",
+			           onErrorArg,
+			           microsecToSec(waitTimeUs),
+			           err3,
+			           fdb_get_error(err3));
+		}
 		auto thisRef = std::static_pointer_cast<BlockingTransactionContext>(shared_from_this());
 		scheduler->schedule([thisRef]() { thisRef->handleOnErrorResult(); });
 	}
@@ -243,8 +322,10 @@ public:
 	AsyncTransactionContext(FDBTransaction* tx,
 	                        std::shared_ptr<ITransactionActor> txActor,
 	                        TTaskFct cont,
-	                        IScheduler* scheduler)
-	  : TransactionContextBase(tx, txActor, cont, scheduler) {}
+	                        IScheduler* scheduler,
+	                        int retryLimit,
+	                        std::string bgBasePath)
+	  : TransactionContextBase(tx, txActor, cont, scheduler, retryLimit, bgBasePath) {}
 
 protected:
 	void doContinueAfter(Future f, TTaskFct cont, bool retryOnError) override {
@@ -252,7 +333,7 @@ protected:
 		if (txState != TxState::IN_PROGRESS) {
 			return;
 		}
-		callbackMap[f.fdbFuture()] = CallbackInfo{ f, cont, shared_from_this(), retryOnError };
+		callbackMap[f.fdbFuture()] = CallbackInfo{ f, cont, shared_from_this(), retryOnError, timeNow() };
 		lock.unlock();
 		fdb_error_t err = fdb_future_set_callback(f.fdbFuture(), futureReadyCallback, this);
 		if (err) {
@@ -264,11 +345,20 @@ protected:
 	}
 
 	static void futureReadyCallback(FDBFuture* f, void* param) {
-		AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
-		txCtx->onFutureReady(f);
+		try {
+			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			txCtx->onFutureReady(f);
+		} catch (std::runtime_error& err) {
+			fmt::print("Unexpected exception in callback {}\n", err.what());
+			abort();
+		} catch (...) {
+			fmt::print("Unknown error in callback\n");
+			abort();
+		}
 	}
 
 	void onFutureReady(FDBFuture* f) {
+		auto endTime = timeNow();
 		injectRandomSleep();
 		// Hold a reference to this to avoid it to be
 		// destroyed before releasing the mutex
@@ -283,6 +373,13 @@ protected:
 		}
 		lock.unlock();
 		fdb_error_t err = fdb_future_get_error(f);
+		auto waitTimeUs = timeElapsedInUs(cbInfo.startTime, endTime);
+		if (waitTimeUs > LONG_WAIT_TIME_US) {
+			fmt::print("Long waiting time on a future: {:.3f}s, return code {} ({})\n",
+			           microsecToSec(waitTimeUs),
+			           err,
+			           fdb_get_error(err));
+		}
 		if (err == error_code_transaction_cancelled) {
 			return;
 		}
@@ -302,8 +399,14 @@ protected:
 		txState = TxState::ON_ERROR;
 		lock.unlock();
 
+		if (!canRetry(err)) {
+			return;
+		}
+
 		ASSERT(!onErrorFuture);
+		onErrorArg = err;
 		onErrorFuture = tx()->onError(err);
+		onErrorCallTimePoint = timeNow();
 		onErrorThisRef = std::static_pointer_cast<AsyncTransactionContext>(shared_from_this());
 		fdb_error_t err2 = fdb_future_set_callback(onErrorFuture.fdbFuture(), onErrorReadyCallback, this);
 		if (err2) {
@@ -313,11 +416,28 @@ protected:
 	}
 
 	static void onErrorReadyCallback(FDBFuture* f, void* param) {
-		AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
-		txCtx->onErrorReady(f);
+		try {
+			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			txCtx->onErrorReady(f);
+		} catch (std::runtime_error& err) {
+			fmt::print("Unexpected exception in callback {}\n", err.what());
+			abort();
+		} catch (...) {
+			fmt::print("Unknown error in callback\n");
+			abort();
+		}
 	}
 
 	void onErrorReady(FDBFuture* f) {
+		auto waitTimeUs = timeElapsedInUs(onErrorCallTimePoint);
+		if (waitTimeUs > LONG_WAIT_TIME_US) {
+			fdb_error_t err = onErrorFuture.getError();
+			fmt::print("Long waiting time on onError({}): {:.3f}s, return code {} ({})\n",
+			           onErrorArg,
+			           microsecToSec(waitTimeUs),
+			           err,
+			           fdb_get_error(err));
+		}
 		injectRandomSleep();
 		auto thisRef = onErrorThisRef;
 		onErrorThisRef = {};
@@ -353,6 +473,7 @@ protected:
 		TTaskFct cont;
 		std::shared_ptr<ITransactionContext> thisRef;
 		bool retryOnError;
+		TimePoint startTime;
 	};
 
 	// Map for keeping track of future waits and holding necessary object references
@@ -369,9 +490,10 @@ class TransactionExecutorBase : public ITransactionExecutor {
 public:
 	TransactionExecutorBase(const TransactionExecutorOptions& options) : options(options), scheduler(nullptr) {}
 
-	void init(IScheduler* scheduler, const char* clusterFile) override {
+	void init(IScheduler* scheduler, const char* clusterFile, const std::string& bgBasePath) override {
 		this->scheduler = scheduler;
 		this->clusterFile = clusterFile;
+		this->bgBasePath = bgBasePath;
 	}
 
 protected:
@@ -385,9 +507,11 @@ protected:
 		} else {
 			std::shared_ptr<ITransactionContext> ctx;
 			if (options.blockOnFutures) {
-				ctx = std::make_shared<BlockingTransactionContext>(tx, txActor, cont, scheduler);
+				ctx = std::make_shared<BlockingTransactionContext>(
+				    tx, txActor, cont, scheduler, options.transactionRetryLimit, bgBasePath);
 			} else {
-				ctx = std::make_shared<AsyncTransactionContext>(tx, txActor, cont, scheduler);
+				ctx = std::make_shared<AsyncTransactionContext>(
+				    tx, txActor, cont, scheduler, options.transactionRetryLimit, bgBasePath);
 			}
 			txActor->init(ctx);
 			txActor->start();
@@ -396,6 +520,7 @@ protected:
 
 protected:
 	TransactionExecutorOptions options;
+	std::string bgBasePath;
 	std::string clusterFile;
 	IScheduler* scheduler;
 };
@@ -409,8 +534,8 @@ public:
 
 	~DBPoolTransactionExecutor() override { release(); }
 
-	void init(IScheduler* scheduler, const char* clusterFile) override {
-		TransactionExecutorBase::init(scheduler, clusterFile);
+	void init(IScheduler* scheduler, const char* clusterFile, const std::string& bgBasePath) override {
+		TransactionExecutorBase::init(scheduler, clusterFile, bgBasePath);
 		for (int i = 0; i < options.numDatabases; i++) {
 			FDBDatabase* db;
 			fdb_error_t err = fdb_create_database(clusterFile, &db);

bindings/c/test/apitester/TesterTransactionExecutor.h

@@ -55,6 +55,9 @@ public:
 	// Mark the transaction as completed without committing it (for read transactions)
 	virtual void done() = 0;
 
+	// Plumbing for blob granule base path
+	virtual std::string getBGBasePath() = 0;
+
 	// A continuation to be executed when all of the given futures get ready
 	virtual void continueAfterAll(std::vector<Future> futures, TTaskFct cont);
 };
@@ -123,6 +126,9 @@ struct TransactionExecutorOptions {
 
 	// The size of the database instance pool
 	int numDatabases = 1;
+
+	// Maximum number of retries per transaction (0 - unlimited)
+	int transactionRetryLimit = 0;
 };
 
 /**
@@ -133,7 +139,7 @@ struct TransactionExecutorOptions {
 class ITransactionExecutor {
 public:
 	virtual ~ITransactionExecutor() {}
-	virtual void init(IScheduler* sched, const char* clusterFile) = 0;
+	virtual void init(IScheduler* sched, const char* clusterFile, const std::string& bgBasePath) = 0;
 	virtual void execute(std::shared_ptr<ITransactionActor> tx, TTaskFct cont) = 0;
 };
 

bindings/c/test/apitester/TesterUtil.cpp

@@ -20,9 +20,18 @@
 
 #include "TesterUtil.h"
 #include <cstdio>
+#include <algorithm>
+#include <ctype.h>
+#include <chrono>
 
 namespace FdbApiTester {
 
+std::string lowerCase(const std::string& str) {
+	std::string res = str;
+	std::transform(res.begin(), res.end(), res.begin(), ::tolower);
+	return res;
+}
+
 Random::Random() {
 	std::random_device dev;
 	random.seed(dev());
@@ -53,6 +62,7 @@ bool Random::randomBool(double trueRatio) {
 
 void print_internal_error(const char* msg, const char* file, int line) {
 	fprintf(stderr, "Assertion %s failed @ %s %d:\n", msg, file, line);
+	fflush(stderr);
 }
 
 } // namespace FdbApiTester

bindings/c/test/apitester/TesterUtil.h

@@ -27,9 +27,11 @@
 #include <ostream>
 #include <optional>
 #include <fmt/format.h>
+#include <chrono>
 
 namespace fmt {
 
+// fmt::format formatting for std::optional<T>
 template <typename T>
 struct formatter<std::optional<T>> : fmt::formatter<T> {
 
@@ -47,6 +49,13 @@ struct formatter<std::optional<T>> : fmt::formatter<T> {
 
 namespace FdbApiTester {
 
+struct KeyValue {
+	std::string key;
+	std::string value;
+};
+
+std::string lowerCase(const std::string& str);
+
 class Random {
 public:
 	Random();
@@ -82,6 +91,25 @@ void print_internal_error(const char* msg, const char* file, int line);
 		}                                                                                                              \
 	} while (false) // For use in destructors, where throwing exceptions is extremely dangerous
 
+using TimePoint = std::chrono::steady_clock::time_point;
+using TimeDuration = std::chrono::microseconds::rep;
+
+static inline TimePoint timeNow() {
+	return std::chrono::steady_clock::now();
+}
+
+static inline TimeDuration timeElapsedInUs(const TimePoint& start, const TimePoint& end) {
+	return std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+}
+
+static inline TimeDuration timeElapsedInUs(const TimePoint& start) {
+	return timeElapsedInUs(start, timeNow());
+}
+
+static inline double microsecToSec(TimeDuration timeUs) {
+	return timeUs / 1000000.0;
+}
+
 } // namespace FdbApiTester
 
 #endif

bindings/c/test/apitester/TesterWorkload.cpp

@@ -20,11 +20,14 @@
 
 #include "TesterWorkload.h"
 #include "TesterUtil.h"
+#include "fmt/core.h"
 #include "test/apitester/TesterScheduler.h"
 #include <cstdlib>
 #include <memory>
 #include <fmt/format.h>
 #include <vector>
+#include <iostream>
+#include <cstdio>
 
 namespace FdbApiTester {
 
@@ -58,6 +61,23 @@ double WorkloadConfig::getFloatOption(const std::string& name, double defaultVal
 	}
 }
 
+bool WorkloadConfig::getBoolOption(const std::string& name, bool defaultVal) const {
+	auto iter = options.find(name);
+	if (iter == options.end()) {
+		return defaultVal;
+	} else {
+		std::string val = lowerCase(iter->second);
+		if (val == "true") {
+			return true;
+		} else if (val == "false") {
+			return false;
+		} else {
+			throw TesterError(
+			    fmt::format("Invalid workload configuration. Invalid value {} for {}", iter->second, name));
+		}
+	}
+}
+
 WorkloadBase::WorkloadBase(const WorkloadConfig& config)
   : manager(nullptr), tasksScheduled(0), numErrors(0), clientId(config.clientId), numClients(config.numClients),
     failed(false) {
@@ -90,7 +110,7 @@ void WorkloadBase::execTransaction(std::shared_ptr<ITransactionActor> tx, TTaskF
 		if (tx->getErrorCode() == error_code_success) {
 			cont();
 		} else {
-			std::string msg = fmt::format("Transaction failed with error: {} ({}})", err, fdb_get_error(err));
+			std::string msg = fmt::format("Transaction failed with error: {} ({})", err, fdb_get_error(err));
 			if (failOnError) {
 				error(msg);
 				failed = true;
@@ -127,9 +147,15 @@ void WorkloadBase::scheduledTaskDone() {
 	}
 }
 
+void WorkloadBase::confirmProgress() {
+	info("Progress confirmed");
+	manager->confirmProgress(this);
+}
+
 void WorkloadManager::add(std::shared_ptr<IWorkload> workload, TTaskFct cont) {
 	std::unique_lock<std::mutex> lock(mutex);
-	workloads[workload.get()] = WorkloadInfo{ workload, cont };
+	workloads[workload.get()] = WorkloadInfo{ workload, cont, workload->getControlIfc(), false };
+	fmt::print(stderr, "Workload {} added\n", workload->getWorkloadId());
 }
 
 void WorkloadManager::run() {
@@ -144,6 +170,13 @@ void WorkloadManager::run() {
 		iter->start();
 	}
 	scheduler->join();
+	if (ctrlInputThread.joinable()) {
+		ctrlInputThread.join();
+	}
+	if (outputPipe.is_open()) {
+		outputPipe << "DONE" << std::endl;
+		outputPipe.close();
+	}
 	if (failed()) {
 		fmt::print(stderr, "{} workloads failed\n", numWorkloadsFailed);
 	} else {
@@ -169,6 +202,90 @@ void WorkloadManager::workloadDone(IWorkload* workload, bool failed) {
 	}
 }
 
+void WorkloadManager::openControlPipes(const std::string& inputPipeName, const std::string& outputPipeName) {
+	if (!inputPipeName.empty()) {
+		ctrlInputThread = std::thread(&WorkloadManager::readControlInput, this, inputPipeName);
+	}
+	if (!outputPipeName.empty()) {
+		fmt::print(stderr, "Opening pipe {} for writing\n", outputPipeName);
+		outputPipe.open(outputPipeName, std::ofstream::out);
+	}
+}
+
+void WorkloadManager::readControlInput(std::string pipeName) {
+	fmt::print(stderr, "Opening pipe {} for reading\n", pipeName);
+	// Open in binary mode and read char-by-char to avoid
+	// any kind of buffering
+	FILE* f = fopen(pipeName.c_str(), "rb");
+	setbuf(f, NULL);
+	std::string line;
+	while (true) {
+		int ch = fgetc(f);
+		if (ch == EOF) {
+			return;
+		}
+		if (ch != '\n') {
+			line += ch;
+			continue;
+		}
+		if (line.empty()) {
+			continue;
+		}
+		fmt::print(stderr, "Received {} command\n", line);
+		if (line == "STOP") {
+			handleStopCommand();
+		} else if (line == "CHECK") {
+			handleCheckCommand();
+		}
+		line.clear();
+	}
+}
+
+void WorkloadManager::handleStopCommand() {
+	std::unique_lock<std::mutex> lock(mutex);
+	for (auto& iter : workloads) {
+		IWorkloadControlIfc* controlIfc = iter.second.controlIfc;
+		if (controlIfc) {
+			controlIfc->stop();
+		}
+	}
+}
+
+void WorkloadManager::handleCheckCommand() {
+	std::unique_lock<std::mutex> lock(mutex);
+	// Request to confirm progress from all workloads
+	// providing the control interface
+	for (auto& iter : workloads) {
+		IWorkloadControlIfc* controlIfc = iter.second.controlIfc;
+		if (controlIfc) {
+			iter.second.progressConfirmed = false;
+			controlIfc->checkProgress();
+		}
+	}
+}
+
+void WorkloadManager::confirmProgress(IWorkload* workload) {
+	std::unique_lock<std::mutex> lock(mutex);
+	// Save the progress confirmation of the workload
+	auto iter = workloads.find(workload);
+	ASSERT(iter != workloads.end());
+	iter->second.progressConfirmed = true;
+	// Check if all workloads have confirmed progress
+	bool allConfirmed = true;
+	for (auto& iter : workloads) {
+		if (iter.second.controlIfc && !iter.second.progressConfirmed) {
+			allConfirmed = false;
+			break;
+		}
+	}
+	lock.unlock();
+	if (allConfirmed) {
+		// Notify the test controller about the successful progress check
+		ASSERT(outputPipe.is_open());
+		outputPipe << "CHECK_OK" << std::endl;
+	}
+}
+
 std::shared_ptr<IWorkload> IWorkloadFactory::create(std::string const& name, const WorkloadConfig& config) {
 	auto it = factories().find(name);
 	if (it == factories().end())

bindings/c/test/apitester/TesterWorkload.h

@@ -29,11 +29,23 @@
 #include <atomic>
 #include <unordered_map>
 #include <mutex>
+#include <thread>
+#include <fstream>
 
 namespace FdbApiTester {
 
 class WorkloadManager;
 
+class IWorkloadControlIfc {
+public:
+	// Stop the workload
+	virtual void stop() = 0;
+
+	// Check if the test is progressing from the point of calling
+	// Progress must be confirmed by calling confirmProgress on the workload manager
+	virtual void checkProgress() = 0;
+};
+
 // Workoad interface
 class IWorkload {
 public:
@@ -44,6 +56,12 @@ public:
 
 	// Start executing the workload
 	virtual void start() = 0;
+
+	// Get workload identifier
+	virtual std::string getWorkloadId() = 0;
+
+	// Get workload control interface if supported, nullptr otherwise
+	virtual IWorkloadControlIfc* getControlIfc() = 0;
 };
 
 // Workload configuration
@@ -57,12 +75,16 @@ struct WorkloadConfig {
 	// Total number of clients
 	int numClients;
 
+	// Selected FDB API version
+	int apiVersion;
+
 	// Workload options: as key-value pairs
 	std::unordered_map<std::string, std::string> options;
 
 	// Get option of a certain type by name. Throws an exception if the values is of a wrong type
 	int getIntOption(const std::string& name, int defaultVal) const;
 	double getFloatOption(const std::string& name, double defaultVal) const;
+	bool getBoolOption(const std::string& name, bool defaultVal) const;
 };
 
 // A base class for test workloads
@@ -74,6 +96,10 @@ public:
 	// Initialize the workload
 	void init(WorkloadManager* manager) override;
 
+	IWorkloadControlIfc* getControlIfc() override { return nullptr; }
+
+	std::string getWorkloadId() override { return workloadId; }
+
 protected:
 	// Schedule the a task as a part of the workload
 	void schedule(TTaskFct task);
@@ -92,6 +118,9 @@ protected:
 	// Log an info message
 	void info(const std::string& msg);
 
+	// Confirm a successfull progress check
+	void confirmProgress();
+
 private:
 	WorkloadManager* manager;
 
@@ -130,6 +159,9 @@ public:
 	WorkloadManager(ITransactionExecutor* txExecutor, IScheduler* scheduler)
 	  : txExecutor(txExecutor), scheduler(scheduler), numWorkloadsFailed(0) {}
 
+	// Open named pipes for communication with the test controller
+	void openControlPipes(const std::string& inputPipeName, const std::string& outputPipeName);
+
 	// Add a workload
 	// A continuation is to be specified for subworkloads
 	void add(std::shared_ptr<IWorkload> workload, TTaskFct cont = NO_OP_TASK);
@@ -152,11 +184,27 @@ private:
 		std::shared_ptr<IWorkload> ref;
 		// Continuation to be executed after completing the workload
 		TTaskFct cont;
+		// Control interface of the workload (optional)
+		IWorkloadControlIfc* controlIfc;
+		// Progress check confirmation status
+		bool progressConfirmed;
 	};
 
 	// To be called by a workload to notify that it is done
 	void workloadDone(IWorkload* workload, bool failed);
 
+	// To be called by a workload to confirm a successful progress check
+	void confirmProgress(IWorkload* workload);
+
+	// Receive and handle control commands from the input pipe
+	void readControlInput(std::string pipeName);
+
+	// Handle STOP command received from the test controller
+	void handleStopCommand();
+
+	// Handle CHECK command received from the test controller
+	void handleCheckCommand();
+
 	// Transaction executor to be used by the workloads
 	ITransactionExecutor* txExecutor;
 
@@ -171,6 +219,12 @@ private:
 
 	// Number of workloads failed
 	int numWorkloadsFailed;
+
+	// Thread for receiving test control commands
+	std::thread ctrlInputThread;
+
+	// Output pipe for emitting test control events
+	std::ofstream outputPipe;
 };
 
 // A workload factory

bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleCorrectnessBlocking.toml

@@ -0,0 +1,24 @@
+[[test]]
+title = 'Blob GranuleAPI Correctness Blocking'
+multiThreaded = true
+buggify = true
+blockOnFutures = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+
+    [[test.workload]]
+    name = 'ApiBlobGranuleCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100

bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleCorrectnessMultiThr.toml

@@ -0,0 +1,23 @@
+[[test]]
+title = 'Blob Granule API Correctness Multi Threaded'
+multiThreaded = true
+buggify = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+    [[test.workload]]
+    name = 'ApiBlobGranuleCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100
+

bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleCorrectnessSingleThr.toml

@@ -0,0 +1,15 @@
+[[test]]
+title = 'Blob Granule API Correctness Single Threaded'
+minClients = 1
+maxClients = 3
+multiThreaded = false
+
+    [[test.workload]]
+    name = 'ApiBlobGranuleCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -45,7 +45,15 @@ enum TesterOptionId {
 	OPT_TRACE_FORMAT,
 	OPT_KNOB,
 	OPT_EXTERNAL_CLIENT_LIBRARY,
-	OPT_TEST_FILE
+	OPT_EXTERNAL_CLIENT_DIRECTORY,
+	OPT_TMP_DIR,
+	OPT_DISABLE_LOCAL_CLIENT,
+	OPT_TEST_FILE,
+	OPT_INPUT_PIPE,
+	OPT_OUTPUT_PIPE,
+	OPT_FDB_API_VERSION,
+	OPT_TRANSACTION_RETRY_LIMIT,
+	OPT_BLOB_GRANULE_LOCAL_FILE_PATH
 };
 
 CSimpleOpt::SOption TesterOptionDefs[] = //
@@ -59,8 +67,16 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
 	  { OPT_KNOB, "--knob-", SO_REQ_SEP },
 	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
+	  { OPT_EXTERNAL_CLIENT_DIRECTORY, "--external-client-dir", SO_REQ_SEP },
+	  { OPT_TMP_DIR, "--tmp-dir", SO_REQ_SEP },
+	  { OPT_DISABLE_LOCAL_CLIENT, "--disable-local-client", SO_NONE },
 	  { OPT_TEST_FILE, "-f", SO_REQ_SEP },
 	  { OPT_TEST_FILE, "--test-file", SO_REQ_SEP },
+	  { OPT_INPUT_PIPE, "--input-pipe", SO_REQ_SEP },
+	  { OPT_OUTPUT_PIPE, "--output-pipe", SO_REQ_SEP },
+	  { OPT_FDB_API_VERSION, "--api-version", SO_REQ_SEP },
+	  { OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP },
+	  { OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP },
 	  SO_END_OF_OPTIONS };
 
 void printProgramUsage(const char* execName) {
@@ -84,9 +100,26 @@ void printProgramUsage(const char* execName) {
 	       "                 Changes a knob option. KNOBNAME should be lowercase.\n"
 	       "  --external-client-library FILE\n"
 	       "                 Path to the external client library.\n"
+	       "  --external-client-dir DIR\n"
+	       "                 Directory containing external client libraries.\n"
+	       "  --tmp-dir DIR\n"
+	       "                 Directory for temporary files of the client.\n"
+	       "  --disable-local-client DIR\n"
+	       "                 Disable the local client, i.e. use only external client libraries.\n"
+	       "  --input-pipe NAME\n"
+	       "                 Name of the input pipe for communication with the test controller.\n"
+	       "  --output-pipe NAME\n"
+	       "                 Name of the output pipe for communication with the test controller.\n"
+	       "  --api-version VERSION\n"
+	       "                 Required FDB API version (default %d).\n"
+	       "  --transaction-retry-limit NUMBER\n"
+	       "				 Maximum number of retries per tranaction (default: 0 - unlimited)\n"
+	       "  --blob-granule-local-file-path PATH\n"
+	       "				 Path to blob granule files on local filesystem\n"
 	       "  -f, --test-file FILE\n"
 	       "                 Test file to run.\n"
-	       "  -h, --help     Display this help and exit.\n");
+	       "  -h, --help     Display this help and exit.\n",
+	       FDB_API_VERSION);
 }
 
 // Extracts the key for command line arguments that are specified with a prefix (e.g. --knob-).
@@ -106,6 +139,19 @@ bool validateTraceFormat(std::string_view format) {
 	return format == "xml" || format == "json";
 }
 
+const int MIN_TESTABLE_API_VERSION = 400;
+
+void processIntOption(const std::string& optionName, const std::string& value, int minValue, int maxValue, int& res) {
+	char* endptr;
+	res = strtol(value.c_str(), &endptr, 10);
+	if (*endptr != '\0') {
+		throw TesterError(fmt::format("Invalid value {} for {}", value, optionName));
+	}
+	if (res < minValue || res > maxValue) {
+		throw TesterError(fmt::format("Value for {} must be between {} and {}", optionName, minValue, maxValue));
+	}
+}
+
 bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 	switch (args.OptionId()) {
 	case OPT_CONNFILE:
@@ -139,11 +185,35 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 	case OPT_EXTERNAL_CLIENT_LIBRARY:
 		options.externalClientLibrary = args.OptionArg();
 		break;
-
+	case OPT_EXTERNAL_CLIENT_DIRECTORY:
+		options.externalClientDir = args.OptionArg();
+		break;
+	case OPT_TMP_DIR:
+		options.tmpDir = args.OptionArg();
+		break;
+	case OPT_DISABLE_LOCAL_CLIENT:
+		options.disableLocalClient = true;
+		break;
 	case OPT_TEST_FILE:
 		options.testFile = args.OptionArg();
 		options.testSpec = readTomlTestSpec(options.testFile);
 		break;
+	case OPT_INPUT_PIPE:
+		options.inputPipeName = args.OptionArg();
+		break;
+	case OPT_OUTPUT_PIPE:
+		options.outputPipeName = args.OptionArg();
+		break;
+	case OPT_FDB_API_VERSION:
+		processIntOption(
+		    args.OptionText(), args.OptionArg(), MIN_TESTABLE_API_VERSION, FDB_API_VERSION, options.apiVersion);
+		break;
+	case OPT_TRANSACTION_RETRY_LIMIT:
+		processIntOption(args.OptionText(), args.OptionArg(), 0, 1000, options.transactionRetryLimit);
+		break;
+	case OPT_BLOB_GRANULE_LOCAL_FILE_PATH:
+		options.bgBasePath = args.OptionArg();
+		break;
 	}
 	return true;
 }
@@ -180,10 +250,23 @@ void fdb_check(fdb_error_t e) {
 }
 
 void applyNetworkOptions(TesterOptions& options) {
+	if (!options.tmpDir.empty()) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_TMP_DIR, options.tmpDir));
+	}
 	if (!options.externalClientLibrary.empty()) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT));
 		fdb_check(
 		    FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY, options.externalClientLibrary));
+	} else if (!options.externalClientDir.empty()) {
+		if (options.disableLocalClient) {
+			fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT));
+		}
+		fdb_check(
+		    FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_DIRECTORY, options.externalClientDir));
+	} else {
+		if (options.disableLocalClient) {
+			throw TesterError("Invalid options: Cannot disable local client if no external library is provided");
+		}
 	}
 
 	if (options.testSpec.multiThreaded) {
@@ -220,34 +303,44 @@ void randomizeOptions(TesterOptions& options) {
 }
 
 bool runWorkloads(TesterOptions& options) {
-	TransactionExecutorOptions txExecOptions;
-	txExecOptions.blockOnFutures = options.testSpec.blockOnFutures;
-	txExecOptions.numDatabases = options.numDatabases;
-	txExecOptions.databasePerTransaction = options.testSpec.databasePerTransaction;
+	try {
+		TransactionExecutorOptions txExecOptions;
+		txExecOptions.blockOnFutures = options.testSpec.blockOnFutures;
+		txExecOptions.numDatabases = options.numDatabases;
+		txExecOptions.databasePerTransaction = options.testSpec.databasePerTransaction;
+		txExecOptions.transactionRetryLimit = options.transactionRetryLimit;
 
-	std::unique_ptr<IScheduler> scheduler = createScheduler(options.numClientThreads);
-	std::unique_ptr<ITransactionExecutor> txExecutor = createTransactionExecutor(txExecOptions);
-	scheduler->start();
-	txExecutor->init(scheduler.get(), options.clusterFile.c_str());
+		std::unique_ptr<IScheduler> scheduler = createScheduler(options.numClientThreads);
+		std::unique_ptr<ITransactionExecutor> txExecutor = createTransactionExecutor(txExecOptions);
+		txExecutor->init(scheduler.get(), options.clusterFile.c_str(), options.bgBasePath);
 
-	WorkloadManager workloadMgr(txExecutor.get(), scheduler.get());
-	for (const auto& workloadSpec : options.testSpec.workloads) {
-		for (int i = 0; i < options.numClients; i++) {
-			WorkloadConfig config;
-			config.name = workloadSpec.name;
-			config.options = workloadSpec.options;
-			config.clientId = i;
-			config.numClients = options.numClients;
-			std::shared_ptr<IWorkload> workload = IWorkloadFactory::create(workloadSpec.name, config);
-			if (!workload) {
-				throw TesterError(fmt::format("Unknown workload '{}'", workloadSpec.name));
+		WorkloadManager workloadMgr(txExecutor.get(), scheduler.get());
+		for (const auto& workloadSpec : options.testSpec.workloads) {
+			for (int i = 0; i < options.numClients; i++) {
+				WorkloadConfig config;
+				config.name = workloadSpec.name;
+				config.options = workloadSpec.options;
+				config.clientId = i;
+				config.numClients = options.numClients;
+				config.apiVersion = options.apiVersion;
+				std::shared_ptr<IWorkload> workload = IWorkloadFactory::create(workloadSpec.name, config);
+				if (!workload) {
+					throw TesterError(fmt::format("Unknown workload '{}'", workloadSpec.name));
+				}
+				workloadMgr.add(workload);
 			}
-			workloadMgr.add(workload);
 		}
-	}
+		if (!options.inputPipeName.empty() || !options.outputPipeName.empty()) {
+			workloadMgr.openControlPipes(options.inputPipeName, options.outputPipeName);
+		}
 
-	workloadMgr.run();
-	return !workloadMgr.failed();
+		scheduler->start();
+		workloadMgr.run();
+		return !workloadMgr.failed();
+	} catch (const std::runtime_error& err) {
+		fmt::print(stderr, "ERROR: {}\n", err.what());
+		return false;
+	}
 }
 
 } // namespace
@@ -264,7 +357,7 @@ int main(int argc, char** argv) {
 		}
 		randomizeOptions(options);
 
-		fdb_check(fdb_select_api_version(options.testSpec.apiVersion));
+		fdb_check(fdb_select_api_version(options.apiVersion));
 		applyNetworkOptions(options);
 		fdb_check(fdb_setup_network());
 

bindings/c/test/apitester/run_c_api_tests.py

@@ -49,10 +49,16 @@ def initialize_logger_level(logging_level):
 
 
 def run_tester(args, test_file):
-    cmd = [args.tester_binary, "--cluster-file",
-           args.cluster_file, "--test-file", test_file]
+    cmd = [args.tester_binary,
+           "--cluster-file", args.cluster_file,
+           "--test-file", test_file]
     if args.external_client_library is not None:
         cmd += ["--external-client-library", args.external_client_library]
+    if args.tmp_dir is not None:
+        cmd += ["--tmp-dir", args.tmp_dir]
+    if args.blob_granule_local_file_path is not None:
+        cmd += ["--blob-granule-local-file-path",
+                args.blob_granule_local_file_path]
 
     get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
     proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
@@ -82,8 +88,8 @@ def run_tester(args, test_file):
 
 def run_tests(args):
     num_failed = 0
-    test_files = [f for f in os.listdir(args.test_dir)
-                  if os.path.isfile(os.path.join(args.test_dir, f)) and f.endswith(".toml")]
+    test_files = [f for f in os.listdir(args.test_dir) if os.path.isfile(
+        os.path.join(args.test_dir, f)) and f.endswith(".toml")]
 
     for test_file in test_files:
         get_logger().info('=========================================================')
@@ -111,6 +117,10 @@ def parse_args(argv):
                         help='The timeout in seconds for running each individual test. (default 300)')
     parser.add_argument('--logging-level', type=str, default='INFO',
                         choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').')
+    parser.add_argument('--tmp-dir', type=str, default=None,
+                        help='The directory for storing temporary files (default: None)')
+    parser.add_argument('--blob-granule-local-file-path', type=str, default=None,
+                        help='Enable blob granule tests if set, value is path to local blob granule files')
 
     return parser.parse_args(argv)
 

bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml

@@ -0,0 +1,35 @@
+[[test]]
+title = 'Mixed Workload for Upgrade Tests with a Multi-Threaded Client'
+multiThreaded = true
+buggify = true
+databasePerTransaction = false
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+    [[test.workload]]
+    name = 'ApiCorrectness'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    runUntilStop = true
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'CancelTransaction'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    runUntilStop = true
+    readExistingKeysRatio = 0.9

bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml

@@ -0,0 +1,33 @@
+[[test]]
+title = 'Mixed Workload for Upgrade Tests with a Single FDB Thread'
+multiThreaded = false
+buggify = true
+databasePerTransaction = false
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+    [[test.workload]]
+    name = 'ApiCorrectness'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    runUntilStop = true
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'CancelTransaction'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    runUntilStop = true
+    readExistingKeysRatio = 0.9

bindings/c/test/unit/unit_tests.cpp

@@ -44,6 +44,8 @@
 #include "fdbclient/Tuple.h"
 
 #include "flow/config.h"
+#include "flow/DeterministicRandom.h"
+#include "flow/IRandom.h"
 
 #include "fdb_api.hpp"
 
@@ -2021,15 +2023,17 @@ TEST_CASE("fdb_transaction_add_conflict_range") {
 TEST_CASE("special-key-space valid transaction ID") {
 	auto value = get_value("\xff\xff/tracing/transaction_id", /* snapshot */ false, {});
 	REQUIRE(value.has_value());
-	uint64_t transaction_id = std::stoul(value.value());
-	CHECK(transaction_id > 0);
+	UID transaction_id = UID::fromString(value.value());
+	CHECK(transaction_id.first() > 0);
+	CHECK(transaction_id.second() > 0);
 }
 
 TEST_CASE("special-key-space custom transaction ID") {
 	fdb::Transaction tr(db);
 	fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
 	while (1) {
-		tr.set("\xff\xff/tracing/transaction_id", std::to_string(ULONG_MAX));
+		UID randomTransactionID = UID(deterministicRandom()->randomUInt64(), deterministicRandom()->randomUInt64());
+		tr.set("\xff\xff/tracing/transaction_id", randomTransactionID.toString());
 		fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/transaction_id",
 		                             /* snapshot */ false);
 
@@ -2046,8 +2050,8 @@ TEST_CASE("special-key-space custom transaction ID") {
 		fdb_check(f1.get(&out_present, (const uint8_t**)&val, &vallen));
 
 		REQUIRE(out_present);
-		uint64_t transaction_id = std::stoul(std::string(val, vallen));
-		CHECK(transaction_id == ULONG_MAX);
+		UID transaction_id = UID::fromString(val);
+		CHECK(transaction_id == randomTransactionID);
 		break;
 	}
 }
@@ -2074,8 +2078,9 @@ TEST_CASE("special-key-space set transaction ID after write") {
 		fdb_check(f1.get(&out_present, (const uint8_t**)&val, &vallen));
 
 		REQUIRE(out_present);
-		uint64_t transaction_id = std::stoul(std::string(val, vallen));
-		CHECK(transaction_id != 0);
+		UID transaction_id = UID::fromString(val);
+		CHECK(transaction_id.first() > 0);
+		CHECK(transaction_id.second() > 0);
 		break;
 	}
 }
@@ -2140,7 +2145,9 @@ TEST_CASE("special-key-space tracing get range") {
 		CHECK(out_count == 2);
 
 		CHECK(std::string((char*)out_kv[1].key, out_kv[1].key_length) == tracingBegin + "transaction_id");
-		CHECK(std::stoul(std::string((char*)out_kv[1].value, out_kv[1].value_length)) > 0);
+		UID transaction_id = UID::fromString(std::string((char*)out_kv[1].value));
+		CHECK(transaction_id.first() > 0);
+		CHECK(transaction_id.second() > 0);
 		break;
 	}
 }

bindings/go/src/fdb/generated.go

@@ -285,6 +285,13 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error {
 	return o.setOpt(90, []byte(param))
 }
 
+// Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp
+//
+// Parameter: Client directory for temporary files. 
+func (o NetworkOptions) SetClientTmpDir(param string) error {
+	return o.setOpt(90, []byte(param))
+}
+
 // Set the size of the client location cache. Raising this value can boost performance in very large databases where clients access data in a near-random pattern. Defaults to 100000.
 //
 // Parameter: Max location cache entries

cmake/AddFdbTest.cmake

@@ -405,6 +405,7 @@ endfunction()
 # Creates a single cluster before running the specified command (usually a ctest test)
 function(add_fdbclient_test)
   set(options DISABLED ENABLED DISABLE_LOG_DUMP)
+  set(options DISABLED ENABLED API_TEST_BLOB_GRANULES_ENABLED)
   set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT WORKING_DIRECTORY)
   set(multiValueArgs COMMAND)
   cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@@ -431,6 +432,9 @@ function(add_fdbclient_test)
   if(T_DISABLE_LOG_DUMP)
     list(APPEND TMP_CLUSTER_CMD --disable-log-dump)
   endif()
+  if(T_API_TEST_BLOB_GRANULES_ENABLED)
+    list(APPEND TMP_CLUSTER_CMD --blob-granules-enabled)
+  endif()
   message(STATUS "Adding Client test ${T_NAME}")
   add_test(NAME "${T_NAME}"
     WORKING_DIRECTORY ${T_WORKING_DIRECTORY}

documentation/sphinx/source/api-version-upgrade-guide.rst

@@ -17,6 +17,8 @@ API version 720
 General
 -------
 
+* Special keys ``\xff\xff/management/profiling/<client_txn_sample_rate|client_txn_size_limit>`` are removed in 7.2 and the functionalities they provide are now covered by the global configuration module.
+
 .. _api-version-upgrade-guide-710:
 
 API version 710

documentation/sphinx/source/client-testing.rst

@@ -113,8 +113,8 @@ Implementing a C++ Workload
 ===========================
 
 In order to implement a workload, one has to build a shared library that links against the fdb client library. This library has to
-exppse a function (with C linkage) called workloadFactory which needs to return a pointer to an object of type ``FDBWorkloadFactory``.
-This mechanism allows the other to implement as many workloads within one library as she wants. To do this the pure virtual classes
+expose a function (with C linkage) called workloadFactory which needs to return a pointer to an object of type ``FDBWorkloadFactory``.
+This mechanism allows the author to implement as many workloads within one library as she wants. To do this the pure virtual classes
 ``FDBWorkloadFactory`` and ``FDBWorkload`` have to be implemented.
 
 .. function:: FDBWorkloadFactory* workloadFactory(FDBLogger*)

documentation/sphinx/source/mr-status-json-schemas.rst.inc

@@ -3,6 +3,40 @@
 .. code-block:: javascript
 
    "cluster":{
+        "storage_wiggler": {
+            "wiggle_server_ids":["0ccb4e0feddb55"],
+            "wiggle_server_addresses": ["127.0.0.1"],
+            "primary": { // primary DC storage wiggler stats
+                // One StorageServer wiggle round is considered 'complete', when all StorageServers with creationTime < T are wiggled
+               "last_round_start_datetime": "2022-04-02 00:05:05.123 +0000",
+               "last_round_start_timestamp": 1648857905.123, // when did the latest round start
+               "last_round_finish_datetime": "1970-01-01 00:00:00.000 +0000",
+               "last_round_finish_timestamp": 0, // when did the latest finished round finish
+               "smoothed_round_seconds": 1, // moving average duration of a wiggle round
+               "finished_round": 1,
+               // 1 wiggle step as 1 storage server is wiggled in the current round
+               "last_wiggle_start_datetime": "2022-04-02 00:05:05.123 +0000",
+               "last_wiggle_start_timestamp": 1648857905.123, // when did the latest wiggle step start
+               "last_wiggle_finish_datetime": "1970-01-01 00:00:00.000 +0000",
+               "last_wiggle_finish_timestamp": 0,
+               "smoothed_wiggle_seconds": 1,
+               "finished_wiggle": 1
+            },
+            "remote": { // remote DC storage wiggler stats
+               "last_round_start_datetime": "2022-04-02 00:05:05.123 +0000",
+               "last_round_start_timestamp": 1648857905.123,
+               "last_round_finish_datetime": "1970-01-01 00:00:00.000 +0000",
+               "last_round_finish_timestamp": 0,
+               "smoothed_round_seconds": 1,
+               "finished_round": 1,
+               "last_wiggle_start_datetime": "2022-04-02 00:05:05.123 +0000",
+               "last_wiggle_start_timestamp": 1648857905.123,
+               "last_wiggle_finish_datetime": "1970-01-01 00:00:00.000 +0000",
+               "last_wiggle_finish_timestamp": 0,
+               "smoothed_wiggle_seconds": 1,
+               "finished_wiggle": 1
+            }
+       },
       "layers":{
          "_valid":true,
          "_error":"some error description"
@@ -702,6 +736,7 @@
              "ssd-2",
              "ssd-redwood-1-experimental",
              "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",
@@ -715,6 +750,7 @@
              "ssd-2",
              "ssd-redwood-1-experimental",
              "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",

documentation/sphinx/source/perpetual-storage-wiggle.rst

@@ -14,7 +14,7 @@ Summary
 ============
 Perpetual storage wiggle is a feature that forces the data distributor to constantly build new storage teams when the cluster is healthy. On a high-level note, the process is like this:
 
-Order storage servers by process id. For each storage server n:
+Order storage servers by their created time, from oldest to newest. For each storage server n:
 
 a. Exclude storage server n.
 
@@ -22,7 +22,7 @@ b. Wait until all data has been moved off the storage server.
 
 c. Include storage n
 
-Goto a to wiggle the next storage process with different process id.
+Goto step a to wiggle the next storage server.
 
 With a perpetual wiggle, storage migrations will be much less impactful. The wiggler will detect the healthy status based on healthy teams, available disk space and the number of unhealthy relocations. It will pause the wiggle until the cluster is healthy again.
 
@@ -47,7 +47,8 @@ Disable perpetual storage wiggle locality matching filter, which wiggles all the
 Monitor
 =======
 
-The ``status`` command in the FDB :ref:`command line interface <command-line-interface>` will show the current perpetual_storage_wiggle value.
+* The ``status`` command will report the IP address of the Storage Server under wiggling.
+* The ``status json`` command in the FDB :ref:`command line interface <command-line-interface>` will show the current ``perpetual_storage_wiggle`` value. Plus, the ``cluster.storage_wiggler`` field reports storage wiggle details.
 
 Trace Events
 ----------------------

documentation/sphinx/source/release-notes/release-notes-710.rst

@@ -23,6 +23,7 @@ Fixes
 
 Status
 ------
+* Added ``cluster.storage_wiggler`` field report storage wiggle stats `(PR #6219) <https://github.com/apple/foundationdb/pull/6219>`_
 
 Bindings
 --------
@@ -33,6 +34,7 @@ Other Changes
 -------------
 * OpenTracing support is now deprecated in favor of OpenTelemetry tracing, which will be enabled in a future release. `(PR #6478) <https://github.com/apple/foundationdb/pull/6478/files>`_
 * Changed ``memory`` option to limit resident memory instead of virtual memory. Added a new ``memory_vsize`` option if limiting virtual memory is desired. `(PR #6719) <https://github.com/apple/foundationdb/pull/6719>`_
+* Change ``perpetual storage wiggle`` to wiggle the storage servers based on their created time. `(PR #6219) <https://github.com/apple/foundationdb/pull/6219>`_
 
 Earlier release notes
 ---------------------

documentation/sphinx/source/special-keys.rst

@@ -191,7 +191,6 @@ that process, and wait for necessary data to be moved away.
 #. ``\xff\xff/management/options/excluded/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/excluded/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 #. ``\xff\xff/management/options/failed/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/failed/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 #. ``\xff\xff/management/min_required_commit_version`` Read/write. Changing this key will change the corresponding system key ``\xff/minRequiredCommitVersion = [[Version]]``. The value of this special key is the literal text of the underlying ``Version``, which is ``int64_t``. If you set the key with a value failed to be parsed as ``int64_t``, ``special_keys_api_failure`` will be thrown. In addition, the given ``Version`` should be larger than the current read version and smaller than the upper bound(``2**63-1-version_per_second*3600*24*365*1000``). Otherwise, ``special_keys_api_failure`` is thrown. For more details, see help text of ``fdbcli`` command ``advanceversion``.
-#. ``\xff\xff/management/profiling/<client_txn_sample_rate|client_txn_size_limit>`` Read/write. Changing these two keys will change the corresponding system keys ``\xff\x02/fdbClientInfo/<client_txn_sample_rate|client_txn_size_limit>``, respectively. The value of ``\xff\xff/management/client_txn_sample_rate`` is a literal text of ``double``, and the value of ``\xff\xff/management/client_txn_size_limit`` is a literal text of ``int64_t``. A special value ``default`` can be set to or read from these two keys, representing the client profiling is disabled. In addition, ``clear`` in this range is not allowed. For more details, see help text of ``fdbcli`` command ``profile client``.
 #. ``\xff\xff/management/maintenance/<zone_id> := <seconds>`` Read/write. Set/clear a key in this range will change the corresponding system key ``\xff\x02/healthyZone``. The value is a literal text of a non-negative ``double`` which represents the remaining time for the zone to be in maintenance. Commiting with an invalid value will throw ``special_keys_api_failure``. Only one zone is allowed to be in maintenance at the same time. Setting a new key in the range will override the old one and the transaction will throw ``special_keys_api_failure`` error if more than one zone is given. For more details, see help text of ``fdbcli`` command ``maintenance``.
    In addition, a special key ``\xff\xff/management/maintenance/IgnoreSSFailures`` in the range, if set, will disable datadistribution for storage server failures.
    It is doing the same thing as the fdbcli command ``datadistribution disable ssfailure``.
@@ -264,6 +263,26 @@ clients can connect FoundationDB transactions to outside events.
 #. ``\xff\xff/tracing/transaction_id := <transaction_id>`` Read/write. A 64-bit integer transaction ID which follows the transaction as it moves through FoundationDB. All transactions are assigned a random transaction ID on creation, and this key can be read to surface the randomly generated ID. Alternatively, set this key to provide a custom identifier. When setting this key, provide a string in the form of a 64-bit integer, which will be automatically converted to the appropriate type.
 #. ``\xff\xff/tracing/token := <tracing_enabled>`` Read/write. Set to true/false to enable or disable tracing for the transaction, respectively. If read, returns a 64-bit integer set to 0 if tracing has been disabled, or a random 64-bit integer otherwise (this integers value has no meaning to the client other than to determine whether the transaction will be traced).
 
+.. _special-key-space-deprecation:
+
+Deprecated Keys
+===============
+
+Listed below are the special keys that have been deprecated. Special key(s) will no longer be accessible when the client specifies an API version equal to or larger than the version where they were deprecated. Clients specifying older API versions will be able to continue using the deprecated key(s).
+
+#. ``\xff\xff/management/profiling/<client_txn_sample_rate|client_txn_size_limit>`` Deprecated as of API version 7.2. The corresponding functionalities are now covered by the global configuration module. For details, see :doc:`global-configuration`. Read/write. Changing these two keys will change the corresponding system keys ``\xff\x02/fdbClientInfo/<client_txn_sample_rate|client_txn_size_limit>``, respectively. The value of ``\xff\xff/management/client_txn_sample_rate`` is a literal text of ``double``, and the value of ``\xff\xff/management/client_txn_size_limit`` is a literal text of ``int64_t``. A special value ``default`` can be set to or read from these two keys, representing the client profiling is disabled. In addition, ``clear`` in this range is not allowed. For more details, see help text of ``fdbcli`` command ``profile client``.
+
+Versioning
+==========
+
+For how FDB clients deal with versioning, see :ref:`api-versions`. The special key space deals with versioning by using the ``API_VERSION`` passed to initialize the client. Any module added at a version larger than the API version set by the client will be inaccessible. For example, if a module is added in version 7.0 and the client sets its API version to 630, then the module will not available. When removing or updating existing modules, module developers need to continue to provide the old behavior for clients that specify old API versions.
+
+To remove the functionality of a certain special key(s), specify the API version where the function is being deprecated in the ``registerSpecialKeysImpl`` function. When a client specifies an API version greater than or equal to the deprecation version, the functionality will not be available. Move and update its documentation to :ref:`special-key-space-deprecation`.
+
+To update the implementation of any special keys, add the new implementation and use ``API_VERSION`` to switch between different implementations.
+
+Add notes in ``api-version-upgrade-guide.rst`` if you either remove or update a special key(s) implementation.
+
 .. [#conflicting_keys] In practice, the transaction probably committed successfully. However, if you're running multiple resolvers then it's possible for a transaction to cause another to abort even if it doesn't commit successfully.
 .. [#max_read_transaction_life_versions] The number 5000000 comes from the server knob MAX_READ_TRANSACTION_LIFE_VERSIONS
 .. [#special_key_space_enable_writes] Enabling this option enables other transaction options, such as ``ACCESS_SYSTEM_KEYS``. This may change in the future.

fdbcli/ConfigureCommand.actor.cpp

@@ -195,6 +195,12 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		fprintf(stderr,
 		        "WARN: RocksDB storage engine type is still in experimental stage, not yet production tested.\n");
 		break;
+	case ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL:
+		printf("Database created\n");
+		fprintf(
+		    stderr,
+		    "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n");
+		break;
 	case ConfigurationResult::DATABASE_UNAVAILABLE:
 		fprintf(stderr, "ERROR: The database is unavailable\n");
 		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
@@ -260,6 +266,12 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		fprintf(stderr,
 		        "WARN: RocksDB storage engine type is still in experimental stage, not yet production tested.\n");
 		break;
+	case ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL:
+		printf("Configuration changed\n");
+		fprintf(
+		    stderr,
+		    "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n");
+		break;
 	default:
 		ASSERT(false);
 		ret = false;

fdbcli/fdbcli.actor.cpp

@@ -1191,6 +1191,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 		ccf = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
 		wait(ccf->resolveHostnames());
 	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled) {
+			throw;
+		}
 		fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
 		return 1;
 	}
@@ -1236,6 +1239,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 			wait(delay(3.0) || success(safeThreadFutureToFuture(tr->getReadVersion())));
 			break;
 		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) {
+				throw;
+			}
 			if (e.code() == error_code_cluster_version_changed) {
 				wait(safeThreadFutureToFuture(tr->onError(e)));
 			} else {
@@ -2023,6 +2029,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 			TraceEvent(SevInfo, "CLICommandLog", randomID).detail("Command", line).detail("IsError", is_error);
 
 		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) {
+				throw;
+			}
 			if (e.code() == error_code_tenant_name_required) {
 				printAtCol("ERROR: tenant name required. Use the `usetenant' command to select a tenant or enable the "
 				           "`RAW_ACCESS' option to read raw keys.",

fdbclient/CommitProxyInterface.h

@@ -162,7 +162,7 @@ struct CommitTransactionRequest : TimedRequest {
 	bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
 
 	Arena arena;
-	SpanID spanContext;
+	SpanContext spanContext;
 	CommitTransactionRef transaction;
 	ReplyPromise<CommitID> reply;
 	uint32_t flags;
@@ -172,8 +172,8 @@ struct CommitTransactionRequest : TimedRequest {
 
 	TenantInfo tenantInfo;
 
-	CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {}
-	CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {}
+	CommitTransactionRequest() : CommitTransactionRequest(SpanContext()) {}
+	CommitTransactionRequest(SpanContext const& context) : spanContext(context), flags(0) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -242,7 +242,7 @@ struct GetReadVersionRequest : TimedRequest {
 		FLAG_PRIORITY_MASK = PRIORITY_SYSTEM_IMMEDIATE,
 	};
 
-	SpanID spanContext;
+	SpanContext spanContext;
 	uint32_t transactionCount;
 	uint32_t flags;
 	TransactionPriority priority;
@@ -255,7 +255,7 @@ struct GetReadVersionRequest : TimedRequest {
 	Version maxVersion; // max version in the client's version vector cache
 
 	GetReadVersionRequest() : transactionCount(1), flags(0), maxVersion(invalidVersion) {}
-	GetReadVersionRequest(SpanID spanContext,
+	GetReadVersionRequest(SpanContext spanContext,
 	                      uint32_t transactionCount,
 	                      TransactionPriority priority,
 	                      Version maxVersion,
@@ -325,7 +325,7 @@ struct GetKeyServerLocationsReply {
 struct GetKeyServerLocationsRequest {
 	constexpr static FileIdentifier file_identifier = 9144680;
 	Arena arena;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Optional<TenantNameRef> tenant;
 	KeyRef begin;
 	Optional<KeyRef> end;
@@ -340,7 +340,7 @@ struct GetKeyServerLocationsRequest {
 	Version minTenantVersion;
 
 	GetKeyServerLocationsRequest() : limit(0), reverse(false), minTenantVersion(latestVersion) {}
-	GetKeyServerLocationsRequest(SpanID spanContext,
+	GetKeyServerLocationsRequest(SpanContext spanContext,
 	                             Optional<TenantNameRef> const& tenant,
 	                             KeyRef const& begin,
 	                             Optional<KeyRef> const& end,
@@ -378,12 +378,12 @@ struct GetRawCommittedVersionReply {
 
 struct GetRawCommittedVersionRequest {
 	constexpr static FileIdentifier file_identifier = 12954034;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Optional<UID> debugID;
 	ReplyPromise<GetRawCommittedVersionReply> reply;
 	Version maxVersion; // max version in the grv proxy's version vector cache
 
-	explicit GetRawCommittedVersionRequest(SpanID spanContext,
+	explicit GetRawCommittedVersionRequest(SpanContext spanContext,
 	                                       Optional<UID> const& debugID = Optional<UID>(),
 	                                       Version maxVersion = invalidVersion)
 	  : spanContext(spanContext), debugID(debugID), maxVersion(maxVersion) {}

fdbclient/CommitTransaction.h

@@ -24,6 +24,7 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Knobs.h"
+#include "flow/Tracing.h"
 
 // The versioned message has wire format : -1, version, messages
 static const int32_t VERSION_HEADER = -1;
@@ -77,6 +78,7 @@ struct MutationRef {
 		AndV2,
 		CompareAndClear,
 		Reserved_For_SpanContextMessage /* See fdbserver/SpanContextMessage.h */,
+		Reserved_For_OTELSpanContextMessage,
 		MAX_ATOMIC_OP
 	};
 	// This is stored this way for serialization purposes.
@@ -190,7 +192,7 @@ struct CommitTransactionRef {
 	Version read_snapshot = 0;
 	bool report_conflicting_keys = false;
 	bool lock_aware = false; // set when metadata mutations are present
-	Optional<SpanID> spanContext;
+	Optional<SpanContext> spanContext;
 
 	template <class Ar>
 	force_inline void serialize(Ar& ar) {

fdbclient/ConfigTransactionInterface.cpp

@@ -34,12 +34,16 @@ void ConfigTransactionInterface::setupWellKnownEndpoints() {
 }
 
 ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote)
-  : getGeneration(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)),
+  : _id(deterministicRandom()->randomUniqueID()),
+    getGeneration(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)),
     get(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GET)),
     getClasses(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
     getKnobs(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)),
     commit(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {}
 
+ConfigTransactionInterface::ConfigTransactionInterface(Hostname const& remote)
+  : _id(deterministicRandom()->randomUniqueID()), hostname(remote) {}
+
 bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rhs) const {
 	return _id == rhs._id;
 }

fdbclient/ConfigTransactionInterface.h

@@ -200,9 +200,12 @@ public:
 	class RequestStream<ConfigTransactionGetKnobsRequest> getKnobs;
 	class RequestStream<ConfigTransactionCommitRequest> commit;
 
+	Optional<Hostname> hostname;
+
 	ConfigTransactionInterface();
 	void setupWellKnownEndpoints();
 	ConfigTransactionInterface(NetworkAddress const& remote);
+	ConfigTransactionInterface(Hostname const& remote);
 
 	bool operator==(ConfigTransactionInterface const& rhs) const;
 	bool operator!=(ConfigTransactionInterface const& rhs) const;
@@ -210,6 +213,6 @@ public:
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, getGeneration, get, getClasses, getKnobs, commit);
+		serializer(ar, getGeneration, get, getClasses, getKnobs, commit, hostname);
 	}
 };

fdbclient/CoordinationInterface.h

@@ -104,6 +104,10 @@ public:
 
 	ConnectionStringStatus status = RESOLVED;
 	AsyncTrigger resolveFinish;
+	// This function tries to resolve all hostnames once, and return them with coords.
+	// Best effort, does not guarantee that the resolves succeed.
+	Future<std::vector<NetworkAddress>> tryResolveHostnames();
+
 	std::vector<NetworkAddress> coords;
 	std::vector<Hostname> hostnames;
 	std::unordered_map<NetworkAddress, Hostname> networkAddressToHostname;

fdbclient/DatabaseConfiguration.cpp

@@ -303,6 +303,9 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
 	           storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
 		result["storage_engine"] = "ssd-rocksdb-v1";
+	} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
+	           storageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
+		result["storage_engine"] = "ssd-sharded-rocksdb";
 	} else if (tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY) {
 		result["storage_engine"] = "memory-1";
 	} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
@@ -325,6 +328,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 			result["tss_storage_engine"] = "ssd-redwood-1-experimental";
 		} else if (testingStorageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
 			result["tss_storage_engine"] = "ssd-rocksdb-v1";
+		} else if (testingStorageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
+			result["tss_storage_engine"] = "ssd-sharded-rocksdb";
 		} else if (testingStorageServerStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
 			result["tss_storage_engine"] = "memory-radixtree-beta";
 		} else if (testingStorageServerStoreType == KeyValueStoreType::MEMORY) {

fdbclient/DatabaseContext.h

@@ -141,7 +141,7 @@ struct WatchParameters : public ReferenceCounted<WatchParameters> {
 
 	const Version version;
 	const TagSet tags;
-	const SpanID spanID;
+	const SpanContext spanContext;
 	const TaskPriority taskID;
 	const Optional<UID> debugID;
 	const UseProvisionalProxies useProvisionalProxies;
@@ -151,11 +151,11 @@ struct WatchParameters : public ReferenceCounted<WatchParameters> {
 	                Optional<Value> value,
 	                Version version,
 	                TagSet tags,
-	                SpanID spanID,
+	                SpanContext spanContext,
 	                TaskPriority taskID,
 	                Optional<UID> debugID,
 	                UseProvisionalProxies useProvisionalProxies)
-	  : tenant(tenant), key(key), value(value), version(version), tags(tags), spanID(spanID), taskID(taskID),
+	  : tenant(tenant), key(key), value(value), version(version), tags(tags), spanContext(spanContext), taskID(taskID),
 	    debugID(debugID), useProvisionalProxies(useProvisionalProxies) {}
 };
 
@@ -416,12 +416,12 @@ public:
 	Optional<TenantName> defaultTenant;
 
 	struct VersionRequest {
-		SpanID spanContext;
+		SpanContext spanContext;
 		Promise<GetReadVersionReply> reply;
 		TagSet tags;
 		Optional<UID> debugID;
 
-		VersionRequest(SpanID spanContext, TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>())
+		VersionRequest(SpanContext spanContext, TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>())
 		  : spanContext(spanContext), tags(tags), debugID(debugID) {}
 	};
 
@@ -594,9 +594,10 @@ public:
 	AsyncTrigger updateCache;
 	std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules;
 	std::unique_ptr<SpecialKeySpace> specialKeySpace;
-	void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module,
-	                                   SpecialKeySpace::IMPLTYPE type,
-	                                   std::unique_ptr<SpecialKeyRangeReadImpl>&& impl);
+	void registerSpecialKeysImpl(SpecialKeySpace::MODULE module,
+	                             SpecialKeySpace::IMPLTYPE type,
+	                             std::unique_ptr<SpecialKeyRangeReadImpl>&& impl,
+	                             int deprecatedVersion = -1);
 
 	static bool debugUseTags;
 	static const std::vector<std::string> debugTransactionTagChoices;

fdbclient/FDBTypes.h

@@ -29,30 +29,10 @@
 #include <unordered_set>
 #include <boost/functional/hash.hpp>
 
-#include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/flow.h"
 
-enum class TraceFlags : uint8_t { unsampled = 0b00000000, sampled = 0b00000001 };
-
-inline TraceFlags operator&(TraceFlags lhs, TraceFlags rhs) {
-	return static_cast<TraceFlags>(static_cast<std::underlying_type_t<TraceFlags>>(lhs) &
-	                               static_cast<std::underlying_type_t<TraceFlags>>(rhs));
-}
-
-struct SpanContext {
-	UID traceID;
-	uint64_t spanID;
-	TraceFlags m_Flags;
-	SpanContext() : traceID(UID()), spanID(0), m_Flags(TraceFlags::unsampled) {}
-	SpanContext(UID traceID, uint64_t spanID, TraceFlags flags) : traceID(traceID), spanID(spanID), m_Flags(flags) {}
-	SpanContext(UID traceID, uint64_t spanID) : traceID(traceID), spanID(spanID), m_Flags(TraceFlags::unsampled) {}
-	SpanContext(Arena arena, const SpanContext& span)
-	  : traceID(span.traceID), spanID(span.spanID), m_Flags(span.m_Flags) {}
-	bool isSampled() const { return (m_Flags & TraceFlags::sampled) == TraceFlags::sampled; }
-};
-
 typedef int64_t Version;
 typedef uint64_t LogEpoch;
 typedef uint64_t Sequence;
@@ -845,7 +825,16 @@ struct KeyValueStoreType {
 	// These enumerated values are stored in the database configuration, so should NEVER be changed.
 	// Only add new ones just before END.
 	// SS storeType is END before the storageServerInterface is initialized.
-	enum StoreType { SSD_BTREE_V1, MEMORY, SSD_BTREE_V2, SSD_REDWOOD_V1, MEMORY_RADIXTREE, SSD_ROCKSDB_V1, END };
+	enum StoreType {
+		SSD_BTREE_V1,
+		MEMORY,
+		SSD_BTREE_V2,
+		SSD_REDWOOD_V1,
+		MEMORY_RADIXTREE,
+		SSD_ROCKSDB_V1,
+		SSD_SHARDED_ROCKSDB,
+		END
+	};
 
 	KeyValueStoreType() : type(END) {}
 	KeyValueStoreType(StoreType type) : type(type) {
@@ -870,6 +859,8 @@ struct KeyValueStoreType {
 			return "ssd-redwood-1-experimental";
 		case SSD_ROCKSDB_V1:
 			return "ssd-rocksdb-v1";
+		case SSD_SHARDED_ROCKSDB:
+			return "ssd-sharded-rocksdb";
 		case MEMORY:
 			return "memory";
 		case MEMORY_RADIXTREE:

fdbclient/GenericManagementAPI.actor.h

@@ -66,7 +66,9 @@ enum class ConfigurationResult {
 	SUCCESS_WARN_PPW_GRADUAL,
 	SUCCESS,
 	SUCCESS_WARN_ROCKSDB_EXPERIMENTAL,
+	SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
 	DATABASE_CREATED_WARN_ROCKSDB_EXPERIMENTAL,
+	DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
 };
 
 enum class CoordinatorsResult {
@@ -293,6 +295,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 	state bool warnPPWGradual = false;
 	state bool warnChangeStorageNoMigrate = false;
 	state bool warnRocksDBIsExperimental = false;
+	state bool warnShardedRocksDBIsExperimental = false;
 	loop {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -483,6 +486,9 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 					} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
 					           newConfig.storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
 						warnRocksDBIsExperimental = true;
+					} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
+					           newConfig.storageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
+						warnShardedRocksDBIsExperimental = true;
 					}
 				}
 			}
@@ -534,6 +540,9 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 						else if (m[configKeysPrefix.toString() + "storage_engine"] ==
 						         std::to_string(KeyValueStoreType::SSD_ROCKSDB_V1))
 							return ConfigurationResult::DATABASE_CREATED_WARN_ROCKSDB_EXPERIMENTAL;
+						else if (m[configKeysPrefix.toString() + "storage_engine"] ==
+						         std::to_string(KeyValueStoreType::SSD_SHARDED_ROCKSDB))
+							return ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
 						else
 							return ConfigurationResult::DATABASE_CREATED;
 					} catch (Error& e2) {
@@ -549,6 +558,8 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 		return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
 	} else if (warnRocksDBIsExperimental) {
 		return ConfigurationResult::SUCCESS_WARN_ROCKSDB_EXPERIMENTAL;
+	} else if (warnShardedRocksDBIsExperimental) {
+		return ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
 	} else {
 		return ConfigurationResult::SUCCESS;
 	}

fdbclient/IClientApi.h

@@ -27,6 +27,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Tenant.h"
 
+#include "flow/Tracing.h"
 #include "flow/ThreadHelper.actor.h"
 
 struct VersionVector;
@@ -96,11 +97,11 @@ public:
 
 	virtual ThreadFuture<Void> commit() = 0;
 	virtual Version getCommittedVersion() = 0;
-	// @todo This API and the "getSpanID()" API may help with debugging simulation
+	// @todo This API and the "getSpanContext()" API may help with debugging simulation
 	// test failures. (These APIs are not currently invoked anywhere.) Remove them
 	// later if they are not really needed.
 	virtual VersionVector getVersionVector() = 0;
-	virtual UID getSpanID() = 0;
+	virtual SpanContext getSpanContext() = 0;
 	virtual ThreadFuture<int64_t> getApproximateSize() = 0;
 
 	virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;

fdbclient/IConfigTransaction.h

@@ -45,7 +45,7 @@ public:
 	// Not implemented:
 	void setVersion(Version) override { throw client_invalid_operation(); }
 	VersionVector getVersionVector() const override { throw client_invalid_operation(); }
-	UID getSpanID() const override { throw client_invalid_operation(); }
+	SpanContext getSpanContext() const override { throw client_invalid_operation(); }
 	Future<Key> getKey(KeySelector const& key, Snapshot snapshot = Snapshot::False) override {
 		throw client_invalid_operation();
 	}

fdbclient/ISingleThreadTransaction.h

@@ -95,7 +95,7 @@ public:
 	virtual Future<Void> commit() = 0;
 	virtual Version getCommittedVersion() const = 0;
 	virtual VersionVector getVersionVector() const = 0;
-	virtual UID getSpanID() const = 0;
+	virtual SpanContext getSpanContext() const = 0;
 	virtual int64_t getApproximateSize() const = 0;
 	virtual Future<Standalone<StringRef>> getVersionstamp() = 0;
 	virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;

fdbclient/ManagementAPI.actor.cpp

@@ -217,6 +217,9 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 	} else if (mode == "ssd-rocksdb-v1") {
 		logType = KeyValueStoreType::SSD_BTREE_V2;
 		storeType = KeyValueStoreType::SSD_ROCKSDB_V1;
+	} else if (mode == "ssd-sharded-rocksdb") {
+		logType = KeyValueStoreType::SSD_BTREE_V2;
+		storeType = KeyValueStoreType::SSD_SHARDED_ROCKSDB;
 	} else if (mode == "memory" || mode == "memory-2") {
 		logType = KeyValueStoreType::SSD_BTREE_V2;
 		storeType = KeyValueStoreType::MEMORY;

fdbclient/MonitorLeader.actor.cpp

@@ -364,6 +364,29 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString") {
 	return Void();
 }
 
+ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnectionString* self) {
+	state std::set<NetworkAddress> allCoordinatorsSet;
+	std::vector<Future<Void>> fs;
+	for (auto& hostname : self->hostnames) {
+		fs.push_back(map(hostname.resolve(), [&](Optional<NetworkAddress> const& addr) -> Void {
+			if (addr.present()) {
+				allCoordinatorsSet.insert(addr.get());
+			}
+			return Void();
+		}));
+	}
+	wait(waitForAll(fs));
+	for (const auto& coord : self->coords) {
+		allCoordinatorsSet.insert(coord);
+	}
+	std::vector<NetworkAddress> allCoordinators(allCoordinatorsSet.begin(), allCoordinatorsSet.end());
+	return allCoordinators;
+}
+
+Future<std::vector<NetworkAddress>> ClusterConnectionString::tryResolveHostnames() {
+	return tryResolveHostnamesImpl(this);
+}
+
 TEST_CASE("/fdbclient/MonitorLeader/PartialResolve") {
 	std::string connectionString = "TestCluster:0@host.name:1234,host-name:5678";
 	std::string hn = "host-name", port = "5678";
@@ -373,19 +396,9 @@ TEST_CASE("/fdbclient/MonitorLeader/PartialResolve") {
 	INetworkConnections::net()->addMockTCPEndpoint(hn, port, { address });
 
 	state ClusterConnectionString cs(connectionString);
-
-	state std::unordered_set<NetworkAddress> coordinatorAddresses;
-	std::vector<Future<Void>> fs;
-	for (auto& hostname : cs.hostnames) {
-		fs.push_back(map(hostname.resolve(), [&](Optional<NetworkAddress> const& addr) -> Void {
-			if (addr.present()) {
-				coordinatorAddresses.insert(addr.get());
-			}
-			return Void();
-		}));
-	}
-	wait(waitForAll(fs));
-	ASSERT(coordinatorAddresses.size() == 1 && coordinatorAddresses.count(address) == 1);
+	state std::vector<NetworkAddress> allCoordinators = wait(cs.tryResolveHostnames());
+	ASSERT(allCoordinators.size() == 1 &&
+	       std::find(allCoordinators.begin(), allCoordinators.end(), address) != allCoordinators.end());
 
 	return Void();
 }
@@ -585,7 +598,7 @@ ACTOR Future<Void> monitorNominee(Key key,
 				    .detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString());
 				if (rep.getError().code() == error_code_request_maybe_delivered) {
 					// Delay to prevent tight resolving loop due to outdated DNS cache
-					wait(delay(FLOW_KNOBS->HOSTNAME_RESOLVE_DELAY));
+					wait(delay(FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL));
 					throw coordinators_changed();
 				} else {
 					throw rep.getError();
@@ -1055,7 +1068,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 
 			auto& ni = rep.get().mutate();
 			shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
-			clientInfo->set(ni);
+			clientInfo->setUnconditional(ni);
 			successIndex = index;
 		} else {
 			TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller

fdbclient/MultiVersionTransaction.actor.cpp

@@ -300,7 +300,9 @@ ThreadResult<RangeResult> DLTransaction::readBlobGranules(const KeyRangeRef& key
 	int count;
 	FdbCApi::fdb_bool_t more;
 	FdbCApi::fdb_error_t error = api->resultGetKeyValueArray(r, &kvs, &count, &more);
-	ASSERT(!error);
+	if (error) {
+		return ThreadResult<RangeResult>(Error(error));
+	}
 
 	// The memory for this is stored in the FDBResult and is released when the result gets destroyed
 	return ThreadResult<RangeResult>(
@@ -1109,13 +1111,13 @@ VersionVector MultiVersionTransaction::getVersionVector() {
 	return VersionVector();
 }
 
-UID MultiVersionTransaction::getSpanID() {
+SpanContext MultiVersionTransaction::getSpanContext() {
 	auto tr = getTransaction();
 	if (tr.transaction) {
-		return tr.transaction->getSpanID();
+		return tr.transaction->getSpanContext();
 	}
 
-	return UID();
+	return SpanContext();
 }
 
 ThreadFuture<int64_t> MultiVersionTransaction::getApproximateSize() {
@@ -1996,8 +1998,9 @@ std::vector<std::pair<std::string, bool>> MultiVersionApi::copyExternalLibraryPe
 		for (int ii = 0; ii < threadCount; ++ii) {
 			std::string filename = basename(path);
 
-			char tempName[PATH_MAX + 12];
-			sprintf(tempName, "/tmp/%s-XXXXXX", filename.c_str());
+			constexpr int MAX_TMP_NAME_LENGTH = PATH_MAX + 12;
+			char tempName[MAX_TMP_NAME_LENGTH];
+			snprintf(tempName, MAX_TMP_NAME_LENGTH, "%s/%s-XXXXXX", tmpDir.c_str(), filename.c_str());
 			int tempFd = mkstemp(tempName);
 			int fd;
 
@@ -2143,6 +2146,9 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		// multiple client threads are not supported on windows.
 		threadCount = extractIntOption(value, 1, 1);
 #endif
+	} else if (option == FDBNetworkOptions::CLIENT_TMP_DIR) {
+		validateOption(value, true, false, false);
+		tmpDir = abspath(value.get().toString());
 	} else {
 		forwardOption = true;
 	}
@@ -2518,7 +2524,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 
 MultiVersionApi::MultiVersionApi()
   : callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false),
-    bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), envOptionsLoaded(false) {}
+    bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), tmpDir("/tmp"),
+    envOptionsLoaded(false) {}
 
 MultiVersionApi* MultiVersionApi::api = new MultiVersionApi();
 

fdbclient/MultiVersionTransaction.h

@@ -378,7 +378,7 @@ public:
 	ThreadFuture<Void> commit() override;
 	Version getCommittedVersion() override;
 	VersionVector getVersionVector() override;
-	UID getSpanID() override { return UID(); };
+	SpanContext getSpanContext() override { return SpanContext(); };
 	ThreadFuture<int64_t> getApproximateSize() override;
 
 	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
@@ -567,7 +567,7 @@ public:
 	ThreadFuture<Void> commit() override;
 	Version getCommittedVersion() override;
 	VersionVector getVersionVector() override;
-	UID getSpanID() override;
+	SpanContext getSpanContext() override;
 	ThreadFuture<int64_t> getApproximateSize() override;
 
 	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
@@ -898,6 +898,7 @@ private:
 
 	int nextThread = 0;
 	int threadCount;
+	std::string tmpDir;
 
 	Mutex lock;
 	std::vector<std::pair<FDBNetworkOptions::Option, Optional<Standalone<StringRef>>>> options;

fdbclient/NativeAPI.actor.cpp

@@ -21,6 +21,7 @@
 #include "fdbclient/NativeAPI.actor.h"
 
 #include <algorithm>
+#include <cstdio>
 #include <iterator>
 #include <regex>
 #include <unordered_set>
@@ -848,7 +849,9 @@ ACTOR Future<Void> assertFailure(GrvProxyInterface remote, Future<ErrorOr<GetRea
 
 Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
                                       std::vector<GrvProxyInterface> newProxies) {
-	Span span(deterministicRandom()->randomUniqueID(), "VerifyCausalReadRisky"_loc);
+	auto debugID = nondeterministicRandom()->randomUniqueID();
+	g_traceBatch.addEvent("AttemptGRVFromOldProxyDebug", debugID.first(), "NativeAPI.attemptGRVFromOldProxies.Start");
+	Span span("VerifyCausalReadRisky"_loc);
 	std::vector<Future<Void>> replies;
 	replies.reserve(oldProxies.size());
 	GetReadVersionRequest req(
@@ -1228,11 +1231,16 @@ Future<HealthMetrics> DatabaseContext::getHealthMetrics(bool detailed = false) {
 	return getHealthMetricsActor(this, detailed);
 }
 
-void DatabaseContext::registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module,
-                                                    SpecialKeySpace::IMPLTYPE type,
-                                                    std::unique_ptr<SpecialKeyRangeReadImpl>&& impl) {
-	specialKeySpace->registerKeyRange(module, type, impl->getKeyRange(), impl.get());
-	specialKeySpaceModules.push_back(std::move(impl));
+// register a special key(s) implementation under the specified module
+void DatabaseContext::registerSpecialKeysImpl(SpecialKeySpace::MODULE module,
+                                              SpecialKeySpace::IMPLTYPE type,
+                                              std::unique_ptr<SpecialKeyRangeReadImpl>&& impl,
+                                              int deprecatedVersion) {
+	// if deprecated, add the implementation when the api version is less than the deprecated version
+	if (deprecatedVersion == -1 || apiVersion < deprecatedVersion) {
+		specialKeySpace->registerKeyRange(module, type, impl->getKeyRange(), impl.get());
+		specialKeySpaceModules.push_back(std::move(impl));
+	}
 }
 
 ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
@@ -1472,188 +1480,188 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 	smoothMidShardSize.reset(CLIENT_KNOBS->INIT_MID_SHARD_BYTES);
 
 	if (apiVersionAtLeast(710)) {
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<TenantMapRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("tenantmap")));
 	}
 	if (apiVersionAtLeast(700)) {
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::ERRORMSG,
-		                              SpecialKeySpace::IMPLTYPE::READONLY,
-		                              std::make_unique<SingleSpecialKeyImpl>(
-		                                  SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin,
-		                                  [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
-			                                  if (ryw->getSpecialKeySpaceErrorMsg().present())
-				                                  return Optional<Value>(ryw->getSpecialKeySpaceErrorMsg().get());
-			                                  else
-				                                  return Optional<Value>();
-		                                  }));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::ERRORMSG,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<SingleSpecialKeyImpl>(
+		                            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin,
+		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
+			                            if (ryw->getSpecialKeySpaceErrorMsg().present())
+				                            return Optional<Value>(ryw->getSpecialKeySpaceErrorMsg().get());
+			                            else
+				                            return Optional<Value>();
+		                            }));
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ManagementCommandsOptionsImpl>(
 		        KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ExcludeServersRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("exclude")));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<FailedServersRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("failed")));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT,
-		                              SpecialKeySpace::IMPLTYPE::READWRITE,
-		                              std::make_unique<ExcludedLocalitiesRangeImpl>(
-		                                  SpecialKeySpace::getManagementApiCommandRange("excludedlocality")));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT,
-		                              SpecialKeySpace::IMPLTYPE::READWRITE,
-		                              std::make_unique<FailedLocalitiesRangeImpl>(
-		                                  SpecialKeySpace::getManagementApiCommandRange("failedlocality")));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::MANAGEMENT,
+		                        SpecialKeySpace::IMPLTYPE::READWRITE,
+		                        std::make_unique<ExcludedLocalitiesRangeImpl>(
+		                            SpecialKeySpace::getManagementApiCommandRange("excludedlocality")));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::MANAGEMENT,
+		                        SpecialKeySpace::IMPLTYPE::READWRITE,
+		                        std::make_unique<FailedLocalitiesRangeImpl>(
+		                            SpecialKeySpace::getManagementApiCommandRange("failedlocality")));
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<ExclusionInProgressRangeImpl>(
 		        KeyRangeRef(LiteralStringRef("in_progress_exclusion/"), LiteralStringRef("in_progress_exclusion0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ProcessClassRangeImpl>(
 		        KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<ProcessClassSourceRangeImpl>(
 		        KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<LockDatabaseImpl>(
 		        singleKeyRange(LiteralStringRef("db_locked"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ConsistencyCheckImpl>(
 		        singleKeyRange(LiteralStringRef("consistency_check_suspended"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::GLOBALCONFIG,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<GlobalConfigImpl>(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::TRACING,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<TracingOptionsImpl>(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::TRACING)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<CoordinatorsImpl>(
 		        KeyRangeRef(LiteralStringRef("coordinators/"), LiteralStringRef("coordinators0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<CoordinatorsAutoImpl>(
 		        singleKeyRange(LiteralStringRef("auto_coordinators"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<AdvanceVersionImpl>(
 		        singleKeyRange(LiteralStringRef("min_required_commit_version"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<VersionEpochImpl>(
 		        singleKeyRange(LiteralStringRef("version_epoch"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ClientProfilingImpl>(
 		        KeyRangeRef(LiteralStringRef("profiling/"), LiteralStringRef("profiling0"))
-		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)),
+		    /* deprecated */ 720);
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<MaintenanceImpl>(
 		        KeyRangeRef(LiteralStringRef("maintenance/"), LiteralStringRef("maintenance0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<DataDistributionImpl>(
 		        KeyRangeRef(LiteralStringRef("data_distribution/"), LiteralStringRef("data_distribution0"))
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::ACTORLINEAGE,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<ActorLineageImpl>(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ACTORLINEAGE)));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF,
-		                              SpecialKeySpace::IMPLTYPE::READWRITE,
-		                              std::make_unique<ActorProfilerConf>(SpecialKeySpace::getModuleRange(
-		                                  SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF)));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF,
+		                        SpecialKeySpace::IMPLTYPE::READWRITE,
+		                        std::make_unique<ActorProfilerConf>(
+		                            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF)));
 	}
 	if (apiVersionAtLeast(630)) {
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION,
-		                              SpecialKeySpace::IMPLTYPE::READONLY,
-		                              std::make_unique<ConflictingKeysImpl>(conflictingKeysRange));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION,
-		                              SpecialKeySpace::IMPLTYPE::READONLY,
-		                              std::make_unique<ReadConflictRangeImpl>(readConflictRangeKeysRange));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION,
-		                              SpecialKeySpace::IMPLTYPE::READONLY,
-		                              std::make_unique<WriteConflictRangeImpl>(writeConflictRangeKeysRange));
-		registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::METRICS,
-		                              SpecialKeySpace::IMPLTYPE::READONLY,
-		                              std::make_unique<DDStatsRangeImpl>(ddStatsRange));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::TRANSACTION,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<ConflictingKeysImpl>(conflictingKeysRange));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::TRANSACTION,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<ReadConflictRangeImpl>(readConflictRangeKeysRange));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::TRANSACTION,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<WriteConflictRangeImpl>(writeConflictRangeKeysRange));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::METRICS,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<DDStatsRangeImpl>(ddStatsRange));
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::METRICS,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<HealthMetricsRangeImpl>(KeyRangeRef(LiteralStringRef("\xff\xff/metrics/health/"),
 		                                                         LiteralStringRef("\xff\xff/metrics/health0"))));
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::WORKERINTERFACE,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<WorkerInterfacesSpecialKeyImpl>(KeyRangeRef(
 		        LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0"))));
-		registerSpecialKeySpaceModule(
-		    SpecialKeySpace::MODULE::STATUSJSON,
-		    SpecialKeySpace::IMPLTYPE::READONLY,
-		    std::make_unique<SingleSpecialKeyImpl>(LiteralStringRef("\xff\xff/status/json"),
-		                                           [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
-			                                           if (ryw->getDatabase().getPtr() &&
-			                                               ryw->getDatabase()->getConnectionRecord()) {
-				                                           ++ryw->getDatabase()->transactionStatusRequests;
-				                                           return getJSON(ryw->getDatabase());
-			                                           } else {
-				                                           return Optional<Value>();
-			                                           }
-		                                           }));
-		registerSpecialKeySpaceModule(
-		    SpecialKeySpace::MODULE::CLUSTERFILEPATH,
-		    SpecialKeySpace::IMPLTYPE::READONLY,
-		    std::make_unique<SingleSpecialKeyImpl>(
-		        LiteralStringRef("\xff\xff/cluster_file_path"),
-		        [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
-			        try {
-				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
-					        Optional<Value> output =
-					            StringRef(ryw->getDatabase()->getConnectionRecord()->getLocation());
-					        return output;
-				        }
-			        } catch (Error& e) {
-				        return e;
-			        }
-			        return Optional<Value>();
-		        }));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::STATUSJSON,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<SingleSpecialKeyImpl>(
+		                            LiteralStringRef("\xff\xff/status/json"),
+		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
+			                            if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
+				                            ++ryw->getDatabase()->transactionStatusRequests;
+				                            return getJSON(ryw->getDatabase());
+			                            } else {
+				                            return Optional<Value>();
+			                            }
+		                            }));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::CLUSTERFILEPATH,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<SingleSpecialKeyImpl>(
+		                            LiteralStringRef("\xff\xff/cluster_file_path"),
+		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
+			                            try {
+				                            if (ryw->getDatabase().getPtr() &&
+				                                ryw->getDatabase()->getConnectionRecord()) {
+					                            Optional<Value> output =
+					                                StringRef(ryw->getDatabase()->getConnectionRecord()->getLocation());
+					                            return output;
+				                            }
+			                            } catch (Error& e) {
+				                            return e;
+			                            }
+			                            return Optional<Value>();
+		                            }));
 
-		registerSpecialKeySpaceModule(
+		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONNECTIONSTRING,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<SingleSpecialKeyImpl>(
@@ -2784,13 +2792,13 @@ void updateTagMappings(Database cx, const GetKeyServerLocationsReply& reply) {
 ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
                                                            Optional<TenantName> tenant,
                                                            Key key,
-                                                           SpanID spanID,
+                                                           SpanContext spanContext,
                                                            Optional<UID> debugID,
                                                            UseProvisionalProxies useProvisionalProxies,
                                                            Reverse isBackward,
                                                            Version version) {
 
-	state Span span("NAPI:getKeyLocation"_loc, spanID);
+	state Span span("NAPI:getKeyLocation"_loc, spanContext);
 	if (isBackward) {
 		ASSERT(key != allKeys.begin && key <= allKeys.end);
 	} else {
@@ -2878,7 +2886,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
                                             Optional<TenantName> const& tenant,
                                             Key const& key,
                                             F StorageServerInterface::*member,
-                                            SpanID spanID,
+                                            SpanContext spanContext,
                                             Optional<UID> debugID,
                                             UseProvisionalProxies useProvisionalProxies,
                                             Reverse isBackward,
@@ -2886,7 +2894,8 @@ Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
 	// we first check whether this range is cached
 	Optional<KeyRangeLocationInfo> locationInfo = cx->getCachedLocation(tenant, key, isBackward);
 	if (!locationInfo.present()) {
-		return getKeyLocation_internal(cx, tenant, key, spanID, debugID, useProvisionalProxies, isBackward, version);
+		return getKeyLocation_internal(
+		    cx, tenant, key, spanContext, debugID, useProvisionalProxies, isBackward, version);
 	}
 
 	bool onlyEndpointFailedAndNeedRefresh = false;
@@ -2900,7 +2909,8 @@ Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
 		cx->invalidateCache(locationInfo.get().tenantEntry.prefix, key);
 
 		// Refresh the cache with a new getKeyLocations made to proxies.
-		return getKeyLocation_internal(cx, tenant, key, spanID, debugID, useProvisionalProxies, isBackward, version);
+		return getKeyLocation_internal(
+		    cx, tenant, key, spanContext, debugID, useProvisionalProxies, isBackward, version);
 	}
 
 	return locationInfo.get();
@@ -2917,7 +2927,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
 	                        useTenant ? trState->tenant() : Optional<TenantName>(),
 	                        key,
 	                        member,
-	                        trState->spanID,
+	                        trState->spanContext,
 	                        trState->debugID,
 	                        trState->useProvisionalProxies,
 	                        isBackward,
@@ -2939,11 +2949,11 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
     KeyRange keys,
     int limit,
     Reverse reverse,
-    SpanID spanID,
+    SpanContext spanContext,
     Optional<UID> debugID,
     UseProvisionalProxies useProvisionalProxies,
     Version version) {
-	state Span span("NAPI:getKeyRangeLocations"_loc, spanID);
+	state Span span("NAPI:getKeyRangeLocations"_loc, spanContext);
 	if (debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.Before");
 
@@ -3013,7 +3023,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& c
                                                                int limit,
                                                                Reverse reverse,
                                                                F StorageServerInterface::*member,
-                                                               SpanID const& spanID,
+                                                               SpanContext const& spanContext,
                                                                Optional<UID> const& debugID,
                                                                UseProvisionalProxies useProvisionalProxies,
                                                                Version version) {
@@ -3023,7 +3033,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& c
 	std::vector<KeyRangeLocationInfo> locations;
 	if (!cx->getCachedLocations(tenant, keys, locations, limit, reverse)) {
 		return getKeyRangeLocations_internal(
-		    cx, tenant, keys, limit, reverse, spanID, debugID, useProvisionalProxies, version);
+		    cx, tenant, keys, limit, reverse, spanContext, debugID, useProvisionalProxies, version);
 	}
 
 	bool foundFailed = false;
@@ -3044,7 +3054,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& c
 	if (foundFailed) {
 		// Refresh the cache with a new getKeyRangeLocations made to proxies.
 		return getKeyRangeLocations_internal(
-		    cx, tenant, keys, limit, reverse, spanID, debugID, useProvisionalProxies, version);
+		    cx, tenant, keys, limit, reverse, spanContext, debugID, useProvisionalProxies, version);
 	}
 
 	return locations;
@@ -3064,7 +3074,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
 	                              limit,
 	                              reverse,
 	                              member,
-	                              trState->spanID,
+	                              trState->spanContext,
 	                              trState->debugID,
 	                              trState->useProvisionalProxies,
 	                              version);
@@ -3093,7 +3103,7 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
 		                                       keys,
 		                                       CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
 		                                       Reverse::False,
-		                                       trState->spanID,
+		                                       trState->spanContext,
 		                                       trState->debugID,
 		                                       trState->useProvisionalProxies,
 		                                       version));
@@ -3124,38 +3134,35 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
 	return Void();
 }
 
-SpanID generateSpanID(bool transactionTracingSample, SpanID parentContext = SpanID()) {
-	uint64_t txnId = deterministicRandom()->randomUInt64();
+SpanContext generateSpanID(bool transactionTracingSample, SpanContext parentContext = SpanContext()) {
 	if (parentContext.isValid()) {
-		if (parentContext.first() > 0) {
-			txnId = parentContext.first();
-		}
-		uint64_t tokenId = parentContext.second() > 0 ? deterministicRandom()->randomUInt64() : 0;
-		return SpanID(txnId, tokenId);
-	} else if (transactionTracingSample) {
-		uint64_t tokenId = deterministicRandom()->random01() <= FLOW_KNOBS->TRACING_SAMPLE_RATE
-		                       ? deterministicRandom()->randomUInt64()
-		                       : 0;
-		return SpanID(txnId, tokenId);
-	} else {
-		return SpanID(txnId, 0);
+		return SpanContext(parentContext.traceID, deterministicRandom()->randomUInt64(), parentContext.m_Flags);
 	}
+	if (transactionTracingSample) {
+		return SpanContext(deterministicRandom()->randomUniqueID(),
+		                   deterministicRandom()->randomUInt64(),
+		                   deterministicRandom()->random01() <= FLOW_KNOBS->TRACING_SAMPLE_RATE
+		                       ? TraceFlags::sampled
+		                       : TraceFlags::unsampled);
+	}
+	return SpanContext(
+	    deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64(), TraceFlags::unsampled);
 }
 
 TransactionState::TransactionState(Database cx,
                                    Optional<TenantName> tenant,
                                    TaskPriority taskID,
-                                   SpanID spanID,
+                                   SpanContext spanContext,
                                    Reference<TransactionLogInfo> trLogInfo)
-  : cx(cx), trLogInfo(trLogInfo), options(cx), taskID(taskID), spanID(spanID), readVersionObtainedFromGrvProxy(true),
-    tenant_(tenant), tenantSet(tenant.present()) {}
+  : cx(cx), trLogInfo(trLogInfo), options(cx), taskID(taskID), spanContext(spanContext),
+    readVersionObtainedFromGrvProxy(true), tenant_(tenant), tenantSet(tenant.present()) {}
 
 Reference<TransactionState> TransactionState::cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo,
                                                             bool generateNewSpan) const {
 
-	SpanID newSpanID = generateNewSpan ? generateSpanID(cx->transactionTracingSample) : spanID;
+	SpanContext newSpanContext = generateNewSpan ? generateSpanID(cx->transactionTracingSample) : spanContext;
 	Reference<TransactionState> newState =
-	    makeReference<TransactionState>(cx, tenant_, cx->taskID, newSpanID, newTrLogInfo);
+	    makeReference<TransactionState>(cx, tenant_, cx->taskID, newSpanContext, newTrLogInfo);
 
 	if (!cx->apiVersionAtLeast(16)) {
 		newState->options = options;
@@ -3213,12 +3220,12 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
                                        UseTenant useTenant,
                                        TransactionRecordLogInfo recordLogInfo) {
 	state Version ver = wait(version);
-	state Span span("NAPI:getValue"_loc, trState->spanID);
+	state Span span("NAPI:getValue"_loc, trState->spanContext);
 	if (useTenant && trState->tenant().present()) {
-		span.addTag("tenant"_sr, trState->tenant().get());
+		span.addAttribute("tenant"_sr, trState->tenant().get());
 	}
 
-	span.addTag("key"_sr, key);
+	span.addAttribute("key"_sr, key);
 	trState->cx->validateVersion(ver);
 
 	loop {
@@ -3344,7 +3351,7 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 	wait(success(version));
 
 	state Optional<UID> getKeyID = Optional<UID>();
-	state Span span("NAPI:getKey"_loc, trState->spanID);
+	state Span span("NAPI:getKey"_loc, trState->spanContext);
 	if (trState->debugID.present()) {
 		getKeyID = nondeterministicRandom()->randomUniqueID();
 
@@ -3443,8 +3450,8 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 	}
 }
 
-ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanID spanContext) {
-	state Span span("NAPI:waitForCommittedVersion"_loc, { spanContext });
+ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanContext spanContext) {
+	state Span span("NAPI:waitForCommittedVersion"_loc, spanContext);
 	try {
 		loop {
 			choose {
@@ -3478,14 +3485,14 @@ ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, Span
 }
 
 ACTOR Future<Version> getRawVersion(Reference<TransactionState> trState) {
-	state Span span("NAPI:getRawVersion"_loc, { trState->spanID });
+	state Span span("NAPI:getRawVersion"_loc, { trState->spanContext });
 	loop {
 		choose {
 			when(wait(trState->cx->onProxiesChanged())) {}
 			when(GetReadVersionReply v =
 			         wait(basicLoadBalance(trState->cx->getGrvProxies(UseProvisionalProxies::False),
 			                               &GrvProxyInterface::getConsistentReadVersion,
-			                               GetReadVersionRequest(trState->spanID,
+			                               GetReadVersionRequest(trState->spanContext,
 			                                                     0,
 			                                                     TransactionPriority::IMMEDIATE,
 			                                                     trState->cx->ssVersionVectorCache.getMaxVersion()),
@@ -3507,7 +3514,7 @@ ACTOR Future<Void> readVersionBatcher(
     uint32_t flags);
 
 ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> parameters) {
-	state Span span("NAPI:watchValue"_loc, parameters->spanID);
+	state Span span("NAPI:watchValue"_loc, parameters->spanContext);
 	state Version ver = parameters->version;
 	cx->validateVersion(parameters->version);
 	ASSERT(parameters->version != latestVersion);
@@ -3517,7 +3524,7 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 		                                                              parameters->tenant.name,
 		                                                              parameters->key,
 		                                                              &StorageServerInterface::watchValue,
-		                                                              parameters->spanID,
+		                                                              parameters->spanContext,
 		                                                              parameters->debugID,
 		                                                              parameters->useProvisionalProxies,
 		                                                              Reverse::False,
@@ -3736,15 +3743,15 @@ ACTOR Future<Void> watchValueMap(Future<Version> version,
                                  Optional<Value> value,
                                  Database cx,
                                  TagSet tags,
-                                 SpanID spanID,
+                                 SpanContext spanContext,
                                  TaskPriority taskID,
                                  Optional<UID> debugID,
                                  UseProvisionalProxies useProvisionalProxies) {
 	state Version ver = wait(version);
 
-	wait(getWatchFuture(
-	    cx,
-	    makeReference<WatchParameters>(tenant, key, value, ver, tags, spanID, taskID, debugID, useProvisionalProxies)));
+	wait(getWatchFuture(cx,
+	                    makeReference<WatchParameters>(
+	                        tenant, key, value, ver, tags, spanContext, taskID, debugID, useProvisionalProxies)));
 
 	return Void();
 }
@@ -3790,10 +3797,11 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
                                         Reverse reverse,
                                         UseTenant useTenant) {
 	state RangeResultFamily output;
-	state Span span("NAPI:getExactRange"_loc, trState->spanID);
+	// TODO - ljoswiak parent or link?
+	state Span span("NAPI:getExactRange"_loc, trState->spanContext);
 
 	if (useTenant && trState->tenant().present()) {
-		span.addTag("tenant"_sr, trState->tenant().get());
+		span.addAttribute("tenant"_sr, trState->tenant().get());
 	}
 
 	// printf("getExactRange( '%s', '%s' )\n", keys.begin.toString().c_str(), keys.end.toString().c_str());
@@ -4150,9 +4158,9 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 	state KeySelector originalBegin = begin;
 	state KeySelector originalEnd = end;
 	state RangeResultFamily output;
-	state Span span("NAPI:getRange"_loc, trState->spanID);
+	state Span span("NAPI:getRange"_loc, trState->spanContext);
 	if (useTenant && trState->tenant().present()) {
-		span.addTag("tenant"_sr, trState->tenant().get());
+		span.addAttribute("tenant"_sr, trState->tenant().get());
 	}
 
 	try {
@@ -4626,7 +4634,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
                                           GetRangeLimits limits,
                                           Snapshot snapshot,
                                           Reverse reverse,
-                                          SpanID spanContext) {
+                                          SpanContext spanContext) {
 	loop {
 		state std::vector<KeyRangeLocationInfo> locations =
 		    wait(getKeyRangeLocations(trState,
@@ -4919,7 +4927,7 @@ ACTOR Future<Void> getRangeStream(Reference<TransactionState> trState,
 
 	// FIXME: better handling to disable row limits
 	ASSERT(!limits.hasRowLimit());
-	state Span span("NAPI:getRangeStream"_loc, trState->spanID);
+	state Span span("NAPI:getRangeStream"_loc, trState->spanContext);
 
 	state Version version = wait(fVersion);
 	trState->cx->validateVersion(version);
@@ -5042,7 +5050,7 @@ Transaction::Transaction(Database const& cx, Optional<TenantName> const& tenant)
                                             cx->taskID,
                                             generateSpanID(cx->transactionTracingSample),
                                             createTrLogInfoProbabilistically(cx))),
-    span(trState->spanID, "Transaction"_loc), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), tr(trState->spanID) {
+    span(trState->spanContext, "Transaction"_loc), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), tr(trState->spanContext) {
 	if (DatabaseContext::debugUseTags) {
 		debugAddTags(trState);
 	}
@@ -5177,7 +5185,7 @@ ACTOR Future<Void> watch(Reference<Watch> watch,
                          Database cx,
                          Future<TenantInfo> tenant,
                          TagSet tags,
-                         SpanID spanID,
+                         SpanContext spanContext,
                          TaskPriority taskID,
                          Optional<UID> debugID,
                          UseProvisionalProxies useProvisionalProxies) {
@@ -5205,7 +5213,7 @@ ACTOR Future<Void> watch(Reference<Watch> watch,
 							                                   watch->value,
 							                                   cx,
 							                                   tags,
-							                                   spanID,
+							                                   spanContext,
 							                                   taskID,
 							                                   debugID,
 							                                   useProvisionalProxies);
@@ -5238,7 +5246,7 @@ Future<Void> Transaction::watch(Reference<Watch> watch) {
 	    populateAndGetTenant(
 	        trState, watch->key, readVersion.isValid() && readVersion.isReady() ? readVersion.get() : latestVersion),
 	    trState->options.readTags,
-	    trState->spanID,
+	    trState->spanContext,
 	    trState->taskID,
 	    trState->debugID,
 	    trState->useProvisionalProxies);
@@ -5726,7 +5734,7 @@ void TransactionOptions::reset(Database const& cx) {
 void Transaction::resetImpl(bool generateNewSpan) {
 	flushTrLogsIfEnabled();
 	trState = trState->cloneAndReset(createTrLogInfoProbabilistically(trState->cx), generateNewSpan);
-	tr = CommitTransactionRequest(trState->spanID);
+	tr = CommitTransactionRequest(trState->spanContext);
 	readVersion = Future<Version>();
 	metadataVersion = Promise<Optional<Key>>();
 	extraConflictRanges.clear();
@@ -5741,7 +5749,7 @@ void Transaction::reset() {
 
 void Transaction::fullReset() {
 	resetImpl(true);
-	span = Span(trState->spanID, "Transaction"_loc);
+	span = Span(trState->spanContext, "Transaction"_loc);
 	backoff = CLIENT_KNOBS->DEFAULT_BACKOFF;
 }
 
@@ -5862,8 +5870,7 @@ ACTOR void checkWrites(Reference<TransactionState> trState,
 ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trState, KeyRange range) {
 	state Transaction tr(trState->cx);
 	state int retries = 0;
-	state Span span("NAPI:dummyTransaction"_loc, trState->spanID);
-	tr.span.addParent(span.context);
+	state Span span(trState->spanContext, "NAPI:dummyTransaction"_loc, span.context);
 	loop {
 		try {
 			TraceEvent("CommitDummyTransaction").detail("Key", range.begin).detail("Retries", retries);
@@ -5906,7 +5913,7 @@ void Transaction::setupWatches() {
 			                                   watches[i]->value,
 			                                   trState->cx,
 			                                   trState->options.readTags,
-			                                   trState->spanID,
+			                                   trState->spanContext,
 			                                   trState->taskID,
 			                                   trState->debugID,
 			                                   trState->useProvisionalProxies));
@@ -6029,7 +6036,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
                                     Future<Version> readVersion) {
 	state TraceInterval interval("TransactionCommit");
 	state double startTime = now();
-	state Span span("NAPI:tryCommit"_loc, trState->spanID);
+	state Span span("NAPI:tryCommit"_loc, trState->spanContext);
 	state Optional<UID> debugID = trState->debugID;
 	if (debugID.present()) {
 		TraceEvent(interval.begin()).detail("Parent", debugID.get());
@@ -6519,10 +6526,11 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::SPAN_PARENT:
 		validateOptionValuePresent(value);
-		if (value.get().size() != 16) {
+		if (value.get().size() != 33) {
 			throw invalid_option_value();
 		}
-		span.addParent(BinaryReader::fromStringRef<UID>(value.get(), Unversioned()));
+		TEST(true); // Adding link in FDBTransactionOptions::SPAN_PARENT
+		span.setParent(BinaryReader::fromStringRef<SpanContext>(value.get(), IncludeVersion()));
 		break;
 
 	case FDBTransactionOptions::REPORT_CONFLICTING_KEYS:
@@ -6565,7 +6573,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 	}
 }
 
-ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanID parentSpan,
+ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpan,
                                                            DatabaseContext* cx,
                                                            uint32_t transactionCount,
                                                            TransactionPriority priority,
@@ -6680,7 +6688,7 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 					}
 					g_traceBatch.addAttach("TransactionAttachID", req.debugID.get().first(), debugID.get().first());
 				}
-				span.addParent(req.spanContext);
+				span.addLink(req.spanContext);
 				requests.push_back(req.reply);
 				for (auto tag : req.tags) {
 					++tags[tag];
@@ -6736,10 +6744,10 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 
 ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
                                          Location location,
-                                         SpanID spanContext,
+                                         SpanContext spanContext,
                                          Future<GetReadVersionReply> f,
                                          Promise<Optional<Value>> metadataVersion) {
-	state Span span(spanContext, location, { trState->spanID });
+	state Span span(spanContext, location, trState->spanContext);
 	GetReadVersionReply rep = wait(f);
 	double replyTime = now();
 	double latency = replyTime - trState->startTime;
@@ -6912,7 +6920,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		}
 
 		Location location = "NAPI:getReadVersion"_loc;
-		UID spanContext = generateSpanID(trState->cx->transactionTracingSample, trState->spanID);
+		SpanContext spanContext = generateSpanID(trState->cx->transactionTracingSample, trState->spanContext);
 		auto const req = DatabaseContext::VersionRequest(spanContext, trState->options.tags, trState->debugID);
 		batcher.stream.send(req);
 		trState->startTime = now();
@@ -7388,7 +7396,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<Transa
                                                                 KeyRange keys,
                                                                 int64_t chunkSize,
                                                                 Version version) {
-	state Span span("NAPI:GetRangeSplitPoints"_loc, trState->spanID);
+	state Span span("NAPI:GetRangeSplitPoints"_loc, trState->spanContext);
 
 	loop {
 		state std::vector<KeyRangeLocationInfo> locations =
@@ -7952,14 +7960,14 @@ Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(cons
 	return Reference<TransactionLogInfo>();
 }
 
-void Transaction::setTransactionID(uint64_t id) {
+void Transaction::setTransactionID(UID id) {
 	ASSERT(getSize() == 0);
-	trState->spanID = SpanID(id, trState->spanID.second());
+	trState->spanContext = SpanContext(id, trState->spanContext.spanID);
 }
 
 void Transaction::setToken(uint64_t token) {
 	ASSERT(getSize() == 0);
-	trState->spanID = SpanID(trState->spanID.first(), token);
+	trState->spanContext = SpanContext(trState->spanContext.traceID, token);
 }
 
 void enableClientInfoLogging() {
@@ -9389,4 +9397,4 @@ ACTOR Future<Void> waitPurgeGranulesCompleteActor(Reference<DatabaseContext> db,
 
 Future<Void> DatabaseContext::waitPurgeGranulesComplete(Key purgeKey) {
 	return waitPurgeGranulesCompleteActor(Reference<DatabaseContext>::addRef(this), purgeKey);
-}
+}

fdbclient/NativeAPI.actor.h

@@ -243,7 +243,7 @@ struct TransactionState : ReferenceCounted<TransactionState> {
 
 	Optional<UID> debugID;
 	TaskPriority taskID;
-	SpanID spanID;
+	SpanContext spanContext;
 	UseProvisionalProxies useProvisionalProxies = UseProvisionalProxies::False;
 	bool readVersionObtainedFromGrvProxy;
 
@@ -259,13 +259,14 @@ struct TransactionState : ReferenceCounted<TransactionState> {
 	std::shared_ptr<CoalescedKeyRangeMap<Value>> conflictingKeys;
 
 	// Only available so that Transaction can have a default constructor, for use in state variables
-	TransactionState(TaskPriority taskID, SpanID spanID) : taskID(taskID), spanID(spanID), tenantSet(false) {}
+	TransactionState(TaskPriority taskID, SpanContext spanContext)
+	  : taskID(taskID), spanContext(spanContext), tenantSet(false) {}
 
 	// VERSION_VECTOR changed default values of readVersionObtainedFromGrvProxy
 	TransactionState(Database cx,
 	                 Optional<TenantName> tenant,
 	                 TaskPriority taskID,
-	                 SpanID spanID,
+	                 SpanContext spanContext,
 	                 Reference<TransactionLogInfo> trLogInfo);
 
 	Reference<TransactionState> cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo, bool generateNewSpan) const;
@@ -435,7 +436,7 @@ public:
 
 	void debugTransaction(UID dID) { trState->debugID = dID; }
 	VersionVector getVersionVector() const;
-	UID getSpanID() const { return trState->spanID; }
+	SpanContext getSpanContext() const { return trState->spanContext; }
 
 	Future<Void> commitMutations();
 	void setupWatches();
@@ -447,7 +448,7 @@ public:
 	Database getDatabase() const { return trState->cx; }
 	static Reference<TransactionLogInfo> createTrLogInfoProbabilistically(const Database& cx);
 
-	void setTransactionID(uint64_t id);
+	void setTransactionID(UID id);
 	void setToken(uint64_t token);
 
 	const std::vector<Future<std::pair<Key, Key>>>& getExtraReadConflictRanges() const { return extraConflictRanges; }
@@ -490,7 +491,7 @@ private:
 	Future<Void> committing;
 };
 
-ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanID spanContext);
+ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanContext spanContext);
 ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
                                                                                   KeyRange keys,
                                                                                   int shardLimit);

fdbclient/ReadYourWrites.actor.cpp

@@ -1982,7 +1982,7 @@ void ReadYourWritesTransaction::getWriteConflicts(KeyRangeMap<bool>* result) {
 	}
 }
 
-void ReadYourWritesTransaction::setTransactionID(uint64_t id) {
+void ReadYourWritesTransaction::setTransactionID(UID id) {
 	tr.setTransactionID(id);
 }
 

fdbclient/ReadYourWrites.h

@@ -140,7 +140,7 @@ public:
 	[[nodiscard]] Future<Void> commit() override;
 	Version getCommittedVersion() const override { return tr.getCommittedVersion(); }
 	VersionVector getVersionVector() const override { return tr.getVersionVector(); }
-	UID getSpanID() const override { return tr.getSpanID(); }
+	SpanContext getSpanContext() const override { return tr.getSpanContext(); }
 
 	int64_t getApproximateSize() const override { return approximateSize; }
 	[[nodiscard]] Future<Standalone<StringRef>> getVersionstamp() override;
@@ -177,7 +177,7 @@ public:
 
 	Reference<const TransactionState> getTransactionState() const { return tr.trState; }
 
-	void setTransactionID(uint64_t id);
+	void setTransactionID(UID id);
 	void setToken(uint64_t token);
 
 	// Read from the special key space readConflictRangeKeysRange

fdbclient/Schemas.cpp

@@ -695,6 +695,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
       },
       "cluster_controller_timestamp":1415650089,
       "protocol_version":"fdb00a400050001",
+      "newest_protocol_version":"fdb00a500040001",
+      "lowest_compatible_protocol_version":"fdb00a500040001",
       "connection_string":"a:a@127.0.0.1:4000",
       "full_replication":true,
       "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece",
@@ -769,6 +771,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
              "ssd-2",
              "ssd-redwood-1-experimental",
              "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",
@@ -782,6 +785,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
              "ssd-2",
              "ssd-redwood-1-experimental",
              "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",

fdbclient/ServerKnobs.cpp

@@ -381,7 +381,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY,                    "fdb");
 
 	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
-	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 					  0.0001 );
+	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE,                    0.0001 );
 	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
 	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
 	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
@@ -393,6 +393,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,              5 );
 	init( ROCKSDB_COMPACTION_READAHEAD_SIZE,                   32768 ); // 32 KB, performs bigger reads when doing compaction.
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
+ 	init( ENABLE_SHARDED_ROCKSDB,                              false );
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
@@ -854,6 +855,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
     init( ENCRYPTION_MODE,                              "AES-256-CTR");
     init( SIM_KMS_MAX_KEYS,                                      4096);
 
+    // Support KmsConnector types are:
+    // KMS_CONNECTOR_TYPE_HTTP -> 1
+    init( KMS_CONNECTOR_TYPE,                      "HttpKmsConnector");
+
 	// Blob granlues
 	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually
 	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (simulationMediumShards || (randomize && BUGGIFY) ) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000; 

fdbclient/ServerKnobs.h

@@ -322,6 +322,7 @@ public:
 	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
+	bool ENABLE_SHARDED_ROCKSDB;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
@@ -821,6 +822,9 @@ public:
 	std::string ENCRYPTION_MODE;
 	int SIM_KMS_MAX_KEYS;
 
+	// Key Management Service (KMS) Connector
+	std::string KMS_CONNECTOR_TYPE;
+
 	// blob granule stuff
 	// FIXME: configure url with database configuration instead of knob eventually
 	std::string BG_URL;

fdbclient/SpecialKeySpace.actor.cpp

@@ -548,6 +548,8 @@ void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module,
                                        SpecialKeySpace::IMPLTYPE type,
                                        const KeyRangeRef& kr,
                                        SpecialKeyRangeReadImpl* impl) {
+	// Not allowed to register an empty range
+	ASSERT(!kr.empty());
 	// module boundary check
 	if (module == SpecialKeySpace::MODULE::TESTONLY) {
 		ASSERT(normalKeys.contains(kr));
@@ -1593,10 +1595,10 @@ Future<RangeResult> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw,
 
 		if (key.endsWith(kTracingTransactionIdKey)) {
 			result.push_back_deep(result.arena(),
-			                      KeyValueRef(key, std::to_string(ryw->getTransactionState()->spanID.first())));
+			                      KeyValueRef(key, ryw->getTransactionState()->spanContext.traceID.toString()));
 		} else if (key.endsWith(kTracingTokenKey)) {
 			result.push_back_deep(result.arena(),
-			                      KeyValueRef(key, std::to_string(ryw->getTransactionState()->spanID.second())));
+			                      KeyValueRef(key, std::to_string(ryw->getTransactionState()->spanContext.spanID)));
 		}
 	}
 	return result;
@@ -1610,7 +1612,7 @@ void TracingOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key,
 	}
 
 	if (key.endsWith(kTracingTransactionIdKey)) {
-		ryw->setTransactionID(std::stoul(value.toString()));
+		ryw->setTransactionID(UID::fromString(value.toString()));
 	} else if (key.endsWith(kTracingTokenKey)) {
 		if (value.toString() == "true") {
 			ryw->setToken(deterministicRandom()->randomUInt64());
@@ -1999,7 +2001,6 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
 	return result;
 }
 
-// TODO : add limitation on set operation
 Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw,
                                                   KeyRangeRef kr,
                                                   GetRangeLimits limitsHint) const {

fdbclient/SpecialKeySpace.actor.h

@@ -485,6 +485,7 @@ public:
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
+// Deprecated as of 7.2
 class ClientProfilingImpl : public SpecialKeyRangeRWImpl {
 public:
 	explicit ClientProfilingImpl(KeyRangeRef kr);

fdbclient/StorageServerInterface.h

@@ -35,6 +35,7 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/TagThrottle.actor.h"
 #include "fdbclient/Tenant.h"
+#include "flow/Tracing.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/VersionVector.h"
 
@@ -271,7 +272,7 @@ struct GetValueReply : public LoadBalancedReply {
 
 struct GetValueRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 8454530;
-	SpanID spanContext;
+	SpanContext spanContext;
 	TenantInfo tenantInfo;
 	Key key;
 	Version version;
@@ -283,7 +284,7 @@ struct GetValueRequest : TimedRequest {
 	                                      // serve the given key
 
 	GetValueRequest() {}
-	GetValueRequest(SpanID spanContext,
+	GetValueRequest(SpanContext spanContext,
 	                const TenantInfo& tenantInfo,
 	                const Key& key,
 	                Version ver,
@@ -315,7 +316,7 @@ struct WatchValueReply {
 
 struct WatchValueRequest {
 	constexpr static FileIdentifier file_identifier = 14747733;
-	SpanID spanContext;
+	SpanContext spanContext;
 	TenantInfo tenantInfo;
 	Key key;
 	Optional<Value> value;
@@ -326,7 +327,7 @@ struct WatchValueRequest {
 
 	WatchValueRequest() {}
 
-	WatchValueRequest(SpanID spanContext,
+	WatchValueRequest(SpanContext spanContext,
 	                  TenantInfo tenantInfo,
 	                  const Key& key,
 	                  Optional<Value> value,
@@ -360,7 +361,7 @@ struct GetKeyValuesReply : public LoadBalancedReply {
 
 struct GetKeyValuesRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Arena arena;
 	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
@@ -418,7 +419,7 @@ struct GetMappedKeyValuesReply : public LoadBalancedReply {
 
 struct GetMappedKeyValuesRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 6795747;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Arena arena;
 	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
@@ -483,7 +484,7 @@ struct GetKeyValuesStreamReply : public ReplyPromiseStreamReply {
 
 struct GetKeyValuesStreamRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Arena arena;
 	TenantInfo tenantInfo;
 	KeySelectorRef begin, end;
@@ -534,7 +535,7 @@ struct GetKeyReply : public LoadBalancedReply {
 
 struct GetKeyRequest : TimedRequest {
 	constexpr static FileIdentifier file_identifier = 10457870;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Arena arena;
 	TenantInfo tenantInfo;
 	KeySelectorRef sel;
@@ -548,7 +549,7 @@ struct GetKeyRequest : TimedRequest {
 
 	GetKeyRequest() {}
 
-	GetKeyRequest(SpanID spanContext,
+	GetKeyRequest(SpanContext spanContext,
 	              TenantInfo tenantInfo,
 	              KeySelectorRef const& sel,
 	              Version version,
@@ -835,7 +836,7 @@ struct ChangeFeedStreamReply : public ReplyPromiseStreamReply {
 
 struct ChangeFeedStreamRequest {
 	constexpr static FileIdentifier file_identifier = 6795746;
-	SpanID spanContext;
+	SpanContext spanContext;
 	Arena arena;
 	Key rangeID;
 	Version begin = 0;

fdbclient/SystemData.cpp

@@ -619,6 +619,19 @@ StorageServerInterface decodeServerListValue(ValueRef const& value) {
 	return decodeServerListValueFB(value);
 }
 
+Value swVersionValue(SWVersion const& swversion) {
+	auto protocolVersion = currentProtocolVersion;
+	protocolVersion.addObjectSerializerFlag();
+	return ObjectWriter::toValue(swversion, IncludeVersion(protocolVersion));
+}
+
+SWVersion decodeSWVersionValue(ValueRef const& value) {
+	SWVersion s;
+	ObjectReader reader(value.begin(), IncludeVersion());
+	reader.deserialize(s);
+	return s;
+}
+
 // processClassKeys.contains(k) iff k.startsWith( processClassKeys.begin ) because '/'+1 == '0'
 const KeyRangeRef processClassKeys(LiteralStringRef("\xff/processClass/"), LiteralStringRef("\xff/processClass0"));
 const KeyRef processClassPrefix = processClassKeys.begin;

fdbclient/SystemData.h

@@ -205,6 +205,9 @@ const Value serverListValue(StorageServerInterface const&);
 UID decodeServerListKey(KeyRef const&);
 StorageServerInterface decodeServerListValue(ValueRef const&);
 
+Value swVersionValue(SWVersion const& swversion);
+SWVersion decodeSWVersionValue(ValueRef const&);
+
 //    "\xff/processClass/[[processID]]" := "[[ProcessClass]]"
 // Contains a mapping from processID to processClass
 extern const KeyRangeRef processClassKeys;

fdbclient/ThreadSafeTransaction.cpp

@@ -465,8 +465,8 @@ VersionVector ThreadSafeTransaction::getVersionVector() {
 	return tr->getVersionVector();
 }
 
-UID ThreadSafeTransaction::getSpanID() {
-	return tr->getSpanID();
+SpanContext ThreadSafeTransaction::getSpanContext() {
+	return tr->getSpanContext();
 }
 
 ThreadFuture<int64_t> ThreadSafeTransaction::getApproximateSize() {

fdbclient/ThreadSafeTransaction.h

@@ -167,7 +167,7 @@ public:
 	ThreadFuture<Void> commit() override;
 	Version getCommittedVersion() override;
 	VersionVector getVersionVector() override;
-	UID getSpanID() override;
+	SpanContext getSpanContext() override;
 	ThreadFuture<int64_t> getApproximateSize() override;
 
 	ThreadFuture<uint64_t> getProtocolVersion();

fdbclient/TransactionLineage.h

@@ -34,10 +34,13 @@ struct TransactionLineage : LineageProperties<TransactionLineage> {
 		GetKeyServersLocations
 	};
 	static constexpr std::string_view name = "Transaction"sv;
-	uint64_t txID;
+	UID txID;
 	Operation operation = Operation::Unset;
 
 	bool isSet(uint64_t TransactionLineage::*member) const { return this->*member > 0; }
+	bool isSet(UID TransactionLineage::*member) const {
+		return static_cast<UID>(this->*member).first() > 0 && static_cast<UID>(this->*member).second() > 0;
+	}
 	bool isSet(Operation TransactionLineage::*member) const { return this->*member != Operation::Unset; }
 };
 

fdbclient/vexillographer/fdb.options

@@ -134,6 +134,9 @@ description is not currently required but encouraged.
     <Option name="distributed_client_tracer" code="90"
             paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, or network_lossy"
             description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
+    <Option name="client_tmp_dir" code="90"
+            paramType="String" paramDescription="Client directory for temporary files. "
+            description="Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp" />
     <Option name="supported_client_versions" code="1000"
             paramType="String" paramDescription="[release version],[source version],[protocol version];..."
             description="This option is set automatically to communicate the list of supported clients to the active client."

fdbrpc/CMakeLists.txt

@@ -35,6 +35,8 @@ set(FDBRPC_SRCS
   sim2.actor.cpp
   sim_validation.cpp
   TimedRequest.h
+  TokenSign.h
+  TokenSign.cpp
   TraceFileIO.cpp
   TSSComparison.h)
 

fdbrpc/FlowTransport.actor.cpp

@@ -406,21 +406,21 @@ TransportData::TransportData(uint64_t transportId, int maxWellKnownEndpoints, IP
 struct ConnectPacket {
 	// The value does not include the size of `connectPacketLength` itself,
 	// but only the other fields of this structure.
-	uint32_t connectPacketLength;
+	uint32_t connectPacketLength = 0;
 	ProtocolVersion protocolVersion; // Expect currentProtocolVersion
 
-	uint16_t canonicalRemotePort; // Port number to reconnect to the originating process
-	uint64_t connectionId; // Multi-version clients will use the same Id for both connections, other connections will
-	                       // set this to zero. Added at protocol Version 0x0FDB00A444020001.
+	uint16_t canonicalRemotePort = 0; // Port number to reconnect to the originating process
+	uint64_t connectionId = 0; // Multi-version clients will use the same Id for both connections, other connections
+	                           // will set this to zero. Added at protocol Version 0x0FDB00A444020001.
 
 	// IP Address to reconnect to the originating process. Only one of these must be populated.
-	uint32_t canonicalRemoteIp4;
+	uint32_t canonicalRemoteIp4 = 0;
 
 	enum ConnectPacketFlags { FLAG_IPV6 = 1 };
-	uint16_t flags;
-	uint8_t canonicalRemoteIp6[16];
+	uint16_t flags = 0;
+	uint8_t canonicalRemoteIp6[16] = { 0 };
 
-	ConnectPacket() { memset(this, 0, sizeof(*this)); }
+	ConnectPacket() = default;
 
 	IPAddress canonicalRemoteIp() const {
 		if (isIPv6()) {

fdbrpc/TokenSign.cpp

@@ -0,0 +1,222 @@
+/*
+ * TokenSign.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/TokenSign.h"
+#include "flow/network.h"
+#include "flow/serialize.h"
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/Platform.h"
+#include "flow/Trace.h"
+#include "flow/UnitTest.h"
+#include <type_traits>
+#include <openssl/ec.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/x509.h>
+
+namespace {
+
+template <typename Func>
+class ExitGuard {
+	std::decay_t<Func> fn;
+
+public:
+	ExitGuard(Func&& fn) : fn(std::forward<Func>(fn)) {}
+
+	~ExitGuard() { fn(); }
+};
+
+[[noreturn]] void traceAndThrow(const char* type) {
+	auto te = TraceEvent(SevWarnAlways, type);
+	te.suppressFor(60);
+	if (auto err = ::ERR_get_error()) {
+		char buf[256]{
+			0,
+		};
+		::ERR_error_string_n(err, buf, sizeof(buf));
+		te.detail("OpenSSLError", buf);
+	}
+	throw digital_signature_ops_error();
+}
+
+struct KeyPairRef {
+	StringRef privateKey;
+	StringRef publicKey;
+};
+
+Standalone<KeyPairRef> generateEcdsaKeyPair() {
+	auto params = std::add_pointer_t<EVP_PKEY>();
+	{
+		auto pctx = ::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr);
+		ASSERT(pctx);
+		auto ctxGuard = ExitGuard([pctx]() { ::EVP_PKEY_CTX_free(pctx); });
+		ASSERT_LT(0, ::EVP_PKEY_paramgen_init(pctx));
+		ASSERT_LT(0, ::EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, NID_X9_62_prime256v1));
+		ASSERT_LT(0, ::EVP_PKEY_paramgen(pctx, &params));
+		ASSERT(params);
+	}
+	auto paramsGuard = ExitGuard([params]() { ::EVP_PKEY_free(params); });
+	// keygen
+	auto kctx = ::EVP_PKEY_CTX_new(params, nullptr);
+	ASSERT(kctx);
+	auto kctxGuard = ExitGuard([kctx]() { ::EVP_PKEY_CTX_free(kctx); });
+	auto key = std::add_pointer_t<EVP_PKEY>();
+	{
+		ASSERT_LT(0, ::EVP_PKEY_keygen_init(kctx));
+		ASSERT_LT(0, ::EVP_PKEY_keygen(kctx, &key));
+	}
+	ASSERT(key);
+	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
+
+	auto ret = Standalone<KeyPairRef>{};
+	auto& arena = ret.arena();
+	{
+		auto len = 0;
+		len = ::i2d_PrivateKey(key, nullptr);
+		ASSERT_LT(0, len);
+		auto buf = new (arena) uint8_t[len];
+		auto out = std::add_pointer_t<uint8_t>(buf);
+		len = ::i2d_PrivateKey(key, &out);
+		ret.privateKey = StringRef(buf, len);
+	}
+	{
+		auto len = 0;
+		len = ::i2d_PUBKEY(key, nullptr);
+		ASSERT_LT(0, len);
+		auto buf = new (arena) uint8_t[len];
+		auto out = std::add_pointer_t<uint8_t>(buf);
+		len = ::i2d_PUBKEY(key, &out);
+		ret.publicKey = StringRef(buf, len);
+	}
+	return ret;
+}
+
+} // namespace
+
+Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName, StringRef privateKeyDer) {
+	auto ret = Standalone<SignedAuthTokenRef>{};
+	auto arena = ret.arena();
+	auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
+	writer.serialize(token);
+	auto tokenStr = writer.toStringRef();
+
+	auto rawPrivKeyDer = privateKeyDer.begin();
+	auto key = ::d2i_AutoPrivateKey(nullptr, &rawPrivKeyDer, privateKeyDer.size());
+	if (!key) {
+		traceAndThrow("SignTokenBadKey");
+	}
+	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
+	auto mdctx = ::EVP_MD_CTX_create();
+	if (!mdctx)
+		traceAndThrow("SignTokenInitFail");
+	auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
+	if (1 != ::EVP_DigestSignInit(mdctx, nullptr, ::EVP_sha256() /*Parameterize?*/, nullptr, key))
+		traceAndThrow("SignTokenInitFail");
+	if (1 != ::EVP_DigestSignUpdate(mdctx, tokenStr.begin(), tokenStr.size()))
+		traceAndThrow("SignTokenUpdateFail");
+	auto sigLen = size_t{};
+	if (1 != ::EVP_DigestSignFinal(mdctx, nullptr, &sigLen)) // assess the length first
+		traceAndThrow("SignTokenGetSigLenFail");
+	auto sigBuf = new (arena) uint8_t[sigLen];
+	if (1 != ::EVP_DigestSignFinal(mdctx, sigBuf, &sigLen))
+		traceAndThrow("SignTokenFinalizeFail");
+	ret.token = tokenStr;
+	ret.signature = StringRef(sigBuf, sigLen);
+	ret.keyName = StringRef(arena, keyName);
+	return ret;
+}
+
+bool verifyToken(SignedAuthTokenRef signedToken, StringRef publicKeyDer) {
+	auto rawPubKeyDer = publicKeyDer.begin();
+	auto key = ::d2i_PUBKEY(nullptr, &rawPubKeyDer, publicKeyDer.size());
+	if (!key)
+		traceAndThrow("VerifyTokenBadKey");
+	auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
+	auto mdctx = ::EVP_MD_CTX_create();
+	if (!mdctx)
+		traceAndThrow("VerifyTokenInitFail");
+	auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
+	if (1 != ::EVP_DigestVerifyInit(mdctx, nullptr, ::EVP_sha256(), nullptr, key))
+		traceAndThrow("VerifyTokenInitFail");
+	if (1 != ::EVP_DigestVerifyUpdate(mdctx, signedToken.token.begin(), signedToken.token.size()))
+		traceAndThrow("VerifyTokenUpdateFail");
+	if (1 != ::EVP_DigestVerifyFinal(mdctx, signedToken.signature.begin(), signedToken.signature.size())) {
+		auto te = TraceEvent(SevInfo, "VerifyTokenFail");
+		te.suppressFor(30);
+		if (auto err = ::ERR_get_error()) {
+			char buf[256]{
+				0,
+			};
+			::ERR_error_string_n(err, buf, sizeof(buf));
+			te.detail("OpenSSLError", buf);
+		}
+		return false;
+	}
+	return true;
+}
+
+void forceLinkTokenSignTests() {}
+
+TEST_CASE("/fdbrpc/TokenSign") {
+	const auto numIters = 100;
+	for (auto i = 0; i < numIters; i++) {
+		auto keyPair = generateEcdsaKeyPair();
+		auto token = Standalone<AuthTokenRef>{};
+		auto arena = token.arena();
+		auto& rng = *deterministicRandom();
+		token.expiresAt = timer_monotonic() * (0.5 + rng.random01());
+		if (auto setIp = rng.randomInt(0, 3)) {
+			if (setIp == 1) {
+				token.ipAddress = IPAddress(rng.randomUInt32());
+			} else {
+				auto v6 = std::array<uint8_t, 16>{};
+				for (auto& byte : v6)
+					byte = rng.randomUInt32() & 255;
+				token.ipAddress = IPAddress(v6);
+			}
+		}
+		auto genRandomStringRef = [&arena, &rng]() {
+			const auto len = rng.randomInt(1, 21);
+			auto strRaw = new (arena) uint8_t[len];
+			for (auto i = 0; i < len; i++)
+				strRaw[i] = (uint8_t)rng.randomAlphaNumeric();
+			return StringRef(strRaw, len);
+		};
+		const auto numTenants = rng.randomInt(0, 31);
+		for (auto i = 0; i < numTenants; i++) {
+			token.tenants.push_back(arena, genRandomStringRef());
+		}
+		auto keyName = genRandomStringRef();
+		auto signedToken = signToken(token, keyName, keyPair.privateKey);
+		const auto verifyExpectOk = verifyToken(signedToken, keyPair.publicKey);
+		ASSERT(verifyExpectOk);
+		// try tampering with signed token by adding one more tenant
+		token.tenants.push_back(arena, genRandomStringRef());
+		auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
+		writer.serialize(token);
+		signedToken.token = writer.toStringRef();
+		const auto verifyExpectFail = verifyToken(signedToken, keyPair.publicKey);
+		ASSERT(!verifyExpectFail);
+	}
+	printf("%d runs OK\n", numIters);
+	return Void();
+}

fdbrpc/TokenSign.h

@@ -0,0 +1,58 @@
+/*
+ * TokenSign.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef FDBRPC_TOKEN_SIGN_H
+#define FDBRPC_TOKEN_SIGN_H
+
+#include "flow/network.h"
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/FileIdentifier.h"
+
+struct AuthTokenRef {
+	static constexpr FileIdentifier file_identifier = 1523118;
+	double expiresAt;
+	IPAddress ipAddress;
+	VectorRef<StringRef> tenants;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, expiresAt, ipAddress, tenants);
+	}
+};
+
+struct SignedAuthTokenRef {
+	static constexpr FileIdentifier file_identifier = 5916732;
+	StringRef token;
+	StringRef keyName;
+	StringRef signature;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, token, keyName, signature);
+	}
+};
+
+Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName, StringRef privateKeyDer);
+
+bool verifyToken(SignedAuthTokenRef signedToken, StringRef publicKeyDer);
+
+#endif // FDBRPC_TOKEN_SIGN_H

fdbrpc/genericactors.actor.h

@@ -73,10 +73,7 @@ Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req, P> to, Req request
 }
 
 ACTOR template <class Req>
-Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
-                                                         Req request,
-                                                         Hostname hostname,
-                                                         WellKnownEndpoints token) {
+Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request, Hostname hostname, WellKnownEndpoints token) {
 	// A wrapper of tryGetReply(request), except that the request is sent to an address resolved from a hostname.
 	// If resolving fails, return lookup_failed().
 	// Otherwise, return tryGetReply(request).
@@ -84,8 +81,8 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
 	if (!address.present()) {
 		return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
 	}
-	*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
-	ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request));
+	RequestStream<Req> to(Endpoint::wellKnown({ address.get() }, token));
+	state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request));
 	if (reply.isError()) {
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
@@ -98,8 +95,7 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
 }
 
 ACTOR template <class Req>
-Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
-                                                         Req request,
+Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
                                                          Hostname hostname,
                                                          WellKnownEndpoints token,
                                                          TaskPriority taskID) {
@@ -110,8 +106,8 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
 	if (!address.present()) {
 		return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
 	}
-	*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
-	ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request, taskID));
+	RequestStream<Req> to(Endpoint::wellKnown({ address.get() }, token));
+	state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request, taskID));
 	if (reply.isError()) {
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
@@ -124,21 +120,21 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
 }
 
 ACTOR template <class Req>
-Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(RequestStream<Req>* to,
-                                                  Req request,
-                                                  Hostname hostname,
-                                                  WellKnownEndpoints token) {
+Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request, Hostname hostname, WellKnownEndpoints token) {
 	// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
 	// Suitable for use with hostname, where RequestStream is NOT initialized yet.
 	// Not normally useful for endpoints initialized with NetworkAddress.
+	state double reconnetInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
 	loop {
 		NetworkAddress address = wait(hostname.resolveWithRetry());
-		*to = RequestStream<Req>(Endpoint::wellKnown({ address }, token));
-		ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request));
+		RequestStream<Req> to(Endpoint::wellKnown({ address }, token));
+		state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request));
 		if (reply.isError()) {
 			resetReply(request);
 			if (reply.getError().code() == error_code_request_maybe_delivered) {
 				// Connection failure.
+				wait(delay(reconnetInterval));
+				reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
 				hostname.resetToUnresolved();
 				INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 			} else {
@@ -151,22 +147,24 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(RequestStream<Req>* to,
 }
 
 ACTOR template <class Req>
-Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(RequestStream<Req>* to,
-                                                  Req request,
+Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
                                                   Hostname hostname,
                                                   WellKnownEndpoints token,
                                                   TaskPriority taskID) {
 	// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
 	// Suitable for use with hostname, where RequestStream is NOT initialized yet.
 	// Not normally useful for endpoints initialized with NetworkAddress.
+	state double reconnetInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
 	loop {
 		NetworkAddress address = wait(hostname.resolveWithRetry());
-		*to = RequestStream<Req>(Endpoint::wellKnown({ address }, token));
-		ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request, taskID));
+		RequestStream<Req> to(Endpoint::wellKnown({ address }, token));
+		state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request, taskID));
 		if (reply.isError()) {
 			resetReply(request);
 			if (reply.getError().code() == error_code_request_maybe_delivered) {
 				// Connection failure.
+				wait(delay(reconnetInterval));
+				reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
 				hostname.resetToUnresolved();
 				INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
 			} else {

fdbserver/ApplyMetadataMutation.cpp

@@ -53,7 +53,7 @@ namespace {
 class ApplyMetadataMutationsImpl {
 
 public:
-	ApplyMetadataMutationsImpl(const SpanID& spanContext_,
+	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           const UID& dbgid_,
 	                           Arena& arena_,
 	                           const VectorRef<MutationRef>& mutations_,
@@ -61,7 +61,7 @@ public:
 	  : spanContext(spanContext_), dbgid(dbgid_), arena(arena_), mutations(mutations_), txnStateStore(txnStateStore_),
 	    confChange(dummyConfChange) {}
 
-	ApplyMetadataMutationsImpl(const SpanID& spanContext_,
+	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           Arena& arena_,
 	                           const VectorRef<MutationRef>& mutations_,
 	                           ProxyCommitData& proxyCommitData_,
@@ -82,7 +82,7 @@ public:
 	    tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap),
 	    initialCommit(initialCommit_) {}
 
-	ApplyMetadataMutationsImpl(const SpanID& spanContext_,
+	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           ResolverData& resolverData_,
 	                           const VectorRef<MutationRef>& mutations_)
 	  : spanContext(spanContext_), dbgid(resolverData_.dbgid), arena(resolverData_.arena), mutations(mutations_),
@@ -94,7 +94,7 @@ public:
 private:
 	// The following variables are incoming parameters
 
-	const SpanID& spanContext;
+	const SpanContext& spanContext;
 
 	const UID& dbgid;
 
@@ -1217,7 +1217,7 @@ public:
 
 } // anonymous namespace
 
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             ProxyCommitData& proxyCommitData,
                             Arena& arena,
                             Reference<ILogSystem> logSystem,
@@ -1241,13 +1241,13 @@ void applyMetadataMutations(SpanID const& spanContext,
 	    .apply();
 }
 
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             ResolverData& resolverData,
                             const VectorRef<MutationRef>& mutations) {
 	ApplyMetadataMutationsImpl(spanContext, resolverData, mutations).apply();
 }
 
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             const UID& dbgid,
                             Arena& arena,
                             const VectorRef<MutationRef>& mutations,

fdbserver/ApplyMetadataMutation.h

@@ -87,7 +87,7 @@ Reference<StorageInfo> getStorageInfo(UID id,
                                       std::map<UID, Reference<StorageInfo>>* storageCache,
                                       IKeyValueStore* txnStateStore);
 
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             ProxyCommitData& proxyCommitData,
                             Arena& arena,
                             Reference<ILogSystem> logSystem,
@@ -97,7 +97,7 @@ void applyMetadataMutations(SpanID const& spanContext,
                             Version version,
                             Version popVersion,
                             bool initialCommit);
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             const UID& dbgid,
                             Arena& arena,
                             const VectorRef<MutationRef>& mutations,
@@ -140,7 +140,7 @@ inline bool containsMetadataMutation(const VectorRef<MutationRef>& mutations) {
 }
 
 // Resolver's version
-void applyMetadataMutations(SpanID const& spanContext,
+void applyMetadataMutations(SpanContext const& spanContext,
                             ResolverData& resolverData,
                             const VectorRef<MutationRef>& mutations);
 

fdbserver/BackupWorker.actor.cpp

@@ -67,6 +67,10 @@ struct VersionedMessage {
 			return false;
 		if (reader.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(reader))
 			return false;
+		if (reader.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(reader)) {
+			TEST(true); // Returning false for OTELSpanContextMessage
+			return false;
+		}
 
 		reader >> *m;
 		return normalKeys.contains(m->param1) || m->param1 == metadataVersionKey;

fdbserver/CMakeLists.txt

@@ -55,6 +55,8 @@ set(FDBSERVER_SRCS
   KeyValueStoreMemory.actor.cpp
   KeyValueStoreRocksDB.actor.cpp
   KeyValueStoreSQLite.actor.cpp
+  KmsConnector.h
+  KmsConnectorInterface.h
   KnobProtectiveGroups.cpp
   KnobProtectiveGroups.h
   Knobs.h
@@ -86,6 +88,7 @@ set(FDBSERVER_SRCS
   OldTLogServer_4_6.actor.cpp
   OldTLogServer_6_0.actor.cpp
   OldTLogServer_6_2.actor.cpp
+  OTELSpanContextMessage.h
   OnDemandStore.actor.cpp
   OnDemandStore.h
   PaxosConfigConsumer.actor.cpp
@@ -133,8 +136,8 @@ set(FDBSERVER_SRCS
   ServerDBInfo.actor.h
   ServerDBInfo.h
   SigStack.cpp
-  SimEncryptKmsProxy.actor.cpp
-  SimEncryptKmsProxy.actor.h
+  SimKmsConnector.actor.h
+  SimKmsConnector.actor.cpp
   SimpleConfigConsumer.actor.cpp
   SimpleConfigConsumer.h
   SimulatedCluster.actor.cpp

fdbserver/ClusterController.actor.cpp

@@ -1097,18 +1097,24 @@ void haltRegisteringOrCurrentSingleton(ClusterControllerData* self,
 	}
 }
 
-void registerWorker(RegisterWorkerRequest req,
-                    ClusterControllerData* self,
-                    std::unordered_set<NetworkAddress> coordinatorAddresses,
-                    ConfigBroadcaster* configBroadcaster) {
+ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
+                                  ClusterControllerData* self,
+                                  ClusterConnectionString cs,
+                                  ConfigBroadcaster* configBroadcaster) {
+	std::vector<NetworkAddress> coordinatorAddresses = wait(cs.tryResolveHostnames());
+
 	const WorkerInterface& w = req.wi;
 	ProcessClass newProcessClass = req.processClass;
 	auto info = self->id_worker.find(w.locality.processId());
 	ClusterControllerPriorityInfo newPriorityInfo = req.priorityInfo;
 	newPriorityInfo.processClassFitness = newProcessClass.machineClassFitness(ProcessClass::ClusterController);
+
 	bool isCoordinator =
-	    (coordinatorAddresses.count(req.wi.address()) > 0) ||
-	    (req.wi.secondaryAddress().present() && coordinatorAddresses.count(req.wi.secondaryAddress().get()) > 0);
+	    (std::find(coordinatorAddresses.begin(), coordinatorAddresses.end(), req.wi.address()) !=
+	     coordinatorAddresses.end()) ||
+	    (req.wi.secondaryAddress().present() &&
+	     std::find(coordinatorAddresses.begin(), coordinatorAddresses.end(), req.wi.secondaryAddress().get()) !=
+	         coordinatorAddresses.end());
 
 	for (auto it : req.incompatiblePeers) {
 		self->db.incompatibleConnections[it] = now() + SERVER_KNOBS->INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
@@ -1271,6 +1277,8 @@ void registerWorker(RegisterWorkerRequest req,
 	if (!req.reply.isSet() && newPriorityInfo != req.priorityInfo) {
 		req.reply.send(RegisterWorkerReply(newProcessClass, newPriorityInfo));
 	}
+
+	return Void();
 }
 
 #define TIME_KEEPER_VERSION LiteralStringRef("1")
@@ -2543,30 +2551,12 @@ ACTOR Future<Void> clusterControllerCore(Reference<IClusterConnectionRecord> con
 		when(RecruitBlobWorkerRequest req = waitNext(interf.recruitBlobWorker.getFuture())) {
 			clusterRecruitBlobWorker(&self, req);
 		}
-		when(state RegisterWorkerRequest req = waitNext(interf.registerWorker.getFuture())) {
+		when(RegisterWorkerRequest req = waitNext(interf.registerWorker.getFuture())) {
 			++self.registerWorkerRequests;
-			state ClusterConnectionString ccs = coordinators.ccr->getConnectionString();
-
-			state std::unordered_set<NetworkAddress> coordinatorAddresses;
-			std::vector<Future<Void>> fs;
-			for (auto& hostname : ccs.hostnames) {
-				fs.push_back(map(hostname.resolve(), [&](Optional<NetworkAddress> const& addr) -> Void {
-					if (addr.present()) {
-						coordinatorAddresses.insert(addr.get());
-					}
-					return Void();
-				}));
-			}
-			wait(waitForAll(fs));
-
-			for (const auto& coord : ccs.coordinators()) {
-				coordinatorAddresses.insert(coord);
-			}
-
-			registerWorker(req,
-			               &self,
-			               coordinatorAddresses,
-			               (configDBType == ConfigDBType::DISABLED) ? nullptr : &configBroadcaster);
+			self.addActor.send(registerWorker(req,
+			                                  &self,
+			                                  coordinators.ccr->getConnectionString(),
+			                                  (configDBType == ConfigDBType::DISABLED) ? nullptr : &configBroadcaster));
 		}
 		when(GetWorkersRequest req = waitNext(interf.getWorkers.getFuture())) {
 			++self.getWorkersRequests;

fdbserver/ClusterRecovery.actor.cpp

@@ -25,6 +25,7 @@
 #include "fdbserver/MasterInterface.h"
 #include "fdbserver/WaitFailure.h"
 
+#include "flow/ProtocolVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 static std::set<int> const& normalClusterRecoveryErrors() {
@@ -1407,6 +1408,11 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	wait(self->cstate.read());
 
+	if (self->cstate.prevDBState.lowestCompatibleProtocolVersion > currentProtocolVersion) {
+		TraceEvent(SevWarnAlways, "IncompatibleProtocolVersion", self->dbgid).log();
+		throw internal_error();
+	}
+
 	self->recoveryState = RecoveryState::LOCKING_CSTATE;
 	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid)
 	    .detail("StatusCode", RecoveryStatus::locking_coordinated_state)
@@ -1462,8 +1468,21 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	DBCoreState newState = self->cstate.myDBState;
 	newState.recoveryCount++;
+	newState.recoveryCount++;
+	if (self->cstate.prevDBState.newestProtocolVersion.isInvalid() ||
+	    self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) {
+		ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalid() ||
+		       !self->cstate.myDBState.newestProtocolVersion.isInvalid());
+		newState.newestProtocolVersion = currentProtocolVersion;
+		newState.lowestCompatibleProtocolVersion = minCompatibleProtocolVersion;
+	}
 	wait(self->cstate.write(newState) || recoverAndEndEpoch);
 
+	TraceEvent("ProtocolVersionCompatibilityChecked", self->dbgid)
+	    .detail("NewestProtocolVersion", self->cstate.myDBState.newestProtocolVersion)
+	    .detail("LowestCompatibleProtocolVersion", self->cstate.myDBState.lowestCompatibleProtocolVersion)
+	    .trackLatest(self->swVersionCheckedEventHolder->trackingKey);
+
 	self->recoveryState = RecoveryState::RECRUITING;
 
 	state std::vector<StorageServerInterface> seedServers;
@@ -1611,7 +1630,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 		tr.set(recoveryCommitRequest.arena, clusterIdKey, BinaryWriter::toValue(self->clusterId, Unversioned()));
 	}
 
-	applyMetadataMutations(SpanID(),
+	applyMetadataMutations(SpanContext(),
 	                       self->dbgid,
 	                       recoveryCommitRequest.arena,
 	                       tr.mutations.slice(mmApplied, tr.mutations.size()),

fdbserver/ClusterRecovery.actor.h

@@ -22,6 +22,7 @@
 
 // When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
 // version.
+#include "flow/Trace.h"
 #include <utility>
 
 #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H)
@@ -244,6 +245,7 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted<ClusterRecoveryData>
 
 	Future<Void> logger;
 
+	Reference<EventCacheHolder> swVersionCheckedEventHolder;
 	Reference<EventCacheHolder> recoveredConfigEventHolder;
 	Reference<EventCacheHolder> clusterRecoveryStateEventHolder;
 	Reference<EventCacheHolder> clusterRecoveryGenerationsEventHolder;
@@ -273,6 +275,7 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted<ClusterRecoveryData>
 	    backupWorkerDoneRequests("BackupWorkerDoneRequests", cc),
 	    getLiveCommittedVersionRequests("GetLiveCommittedVersionRequests", cc),
 	    reportLiveCommittedVersionRequests("ReportLiveCommittedVersionRequests", cc),
+	    swVersionCheckedEventHolder(makeReference<EventCacheHolder>("SWVersionCompatibilityChecked")),
 	    recoveredConfigEventHolder(makeReference<EventCacheHolder>("RecoveredConfig")) {
 		clusterRecoveryStateEventHolder = makeReference<EventCacheHolder>(
 		    getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME));

fdbserver/CommitProxyServer.actor.cpp

@@ -358,7 +358,7 @@ ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
 	state int yieldBytes = 0;
 	state BinaryWriter valueWriter(Unversioned());
 
-	toCommit->addTransactionInfo(SpanID());
+	toCommit->addTransactionInfo(SpanContext());
 
 	// Serialize the log range mutations within the map
 	for (; logRangeMutation != logRangeMutations->cend(); ++logRangeMutation) {
@@ -654,7 +654,7 @@ void CommitBatchContext::setupTraceBatch() {
 
 			g_traceBatch.addAttach("CommitAttachID", tr.debugID.get().first(), debugID.get().first());
 		}
-		span.addParent(tr.spanContext);
+		span.addLink(tr.spanContext);
 	}
 
 	if (debugID.present()) {
@@ -880,7 +880,7 @@ void applyMetadataEffect(CommitBatchContext* self) {
 				committed =
 				    committed && self->resolution[resolver].stateMutations[versionIndex][transactionIndex].committed;
 			if (committed) {
-				applyMetadataMutations(SpanID(),
+				applyMetadataMutations(SpanContext(),
 				                       *self->pProxyCommitData,
 				                       self->arena,
 				                       self->pProxyCommitData->logSystem,
@@ -1300,8 +1300,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 			// simulation
 			TEST(true); // Semi-committed pipeline limited by MVCC window
 			//TraceEvent("ProxyWaitingForCommitted", pProxyCommitData->dbgid).detail("CommittedVersion", pProxyCommitData->committedVersion.get()).detail("NeedToCommit", commitVersion);
-			waitVersionSpan = Span(
-			    deterministicRandom()->randomUniqueID(), "MP:overMaxReadTransactionLifeVersions"_loc, { span.context });
+			waitVersionSpan = Span("MP:overMaxReadTransactionLifeVersions"_loc, span.context);
 			choose {
 				when(wait(pProxyCommitData->committedVersion.whenAtLeast(
 				    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS))) {
@@ -1681,7 +1680,7 @@ void addTagMapping(GetKeyServerLocationsReply& reply, ProxyCommitData* commitDat
 ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsRequest req, ProxyCommitData* commitData) {
 	// We can't respond to these requests until we have valid txnStateStore
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyServersLocations;
-	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
+	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 	wait(commitData->validState.getFuture());
 	wait(delay(0, TaskPriority::DefaultEndpoint));
 
@@ -2201,7 +2200,7 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 
 		Arena arena;
 		bool confChanges;
-		applyMetadataMutations(SpanID(),
+		applyMetadataMutations(SpanContext(),
 		                       *pContext->pCommitData,
 		                       arena,
 		                       Reference<ILogSystem>(),

fdbserver/Coordination.actor.cpp

@@ -75,7 +75,7 @@ struct GenerationRegVal {
 	}
 };
 
-GenerationRegInterface::GenerationRegInterface(NetworkAddress remote)
+GenerationRegInterface::GenerationRegInterface(NetworkAddress const& remote)
   : read(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_READ)),
     write(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_WRITE)) {}
 
@@ -84,7 +84,7 @@ GenerationRegInterface::GenerationRegInterface(INetwork* local) {
 	write.makeWellKnownEndpoint(WLTOKEN_GENERATIONREG_WRITE, TaskPriority::Coordination);
 }
 
-LeaderElectionRegInterface::LeaderElectionRegInterface(NetworkAddress remote)
+LeaderElectionRegInterface::LeaderElectionRegInterface(NetworkAddress const& remote)
   : ClientLeaderRegInterface(remote), candidacy(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_CANDIDACY)),
     electionResult(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT)),
     leaderHeartbeat(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT)),

fdbserver/CoordinationInterface.h

@@ -53,9 +53,9 @@ struct GenerationRegInterface {
 	// the v2 of the previous generation is the v1 of the next.
 
 	GenerationRegInterface() {}
-	GenerationRegInterface(NetworkAddress remote);
+	GenerationRegInterface(NetworkAddress const& remote);
 	GenerationRegInterface(INetwork* local);
-	GenerationRegInterface(Hostname hostname) : hostname(hostname){};
+	GenerationRegInterface(Hostname const& hostname) : hostname(hostname){};
 };
 
 struct UniqueGeneration {
@@ -128,9 +128,9 @@ struct LeaderElectionRegInterface : ClientLeaderRegInterface {
 	RequestStream<struct ForwardRequest> forward;
 
 	LeaderElectionRegInterface() {}
-	LeaderElectionRegInterface(NetworkAddress remote);
+	LeaderElectionRegInterface(NetworkAddress const& remote);
 	LeaderElectionRegInterface(INetwork* local);
-	LeaderElectionRegInterface(Hostname hostname) : ClientLeaderRegInterface(hostname) {}
+	LeaderElectionRegInterface(Hostname const& hostname) : ClientLeaderRegInterface(hostname) {}
 };
 
 struct CandidacyRequest {
@@ -220,7 +220,7 @@ class ConfigNode;
 
 class ServerCoordinators : public ClientCoordinators {
 public:
-	explicit ServerCoordinators(Reference<IClusterConnectionRecord>);
+	explicit ServerCoordinators(Reference<IClusterConnectionRecord> ccr);
 
 	std::vector<LeaderElectionRegInterface> leaderElectionServers;
 	std::vector<GenerationRegInterface> stateServers;

fdbserver/DBCoreState.h

@@ -141,8 +141,13 @@ struct DBCoreState {
 	DBRecoveryCount recoveryCount; // Increases with sequential successful recoveries.
 	LogSystemType logSystemType;
 	std::set<int8_t> pseudoLocalities;
+	ProtocolVersion newestProtocolVersion;
+	ProtocolVersion lowestCompatibleProtocolVersion;
 
-	DBCoreState() : logRouterTags(0), txsTags(0), recoveryCount(0), logSystemType(LogSystemType::empty) {}
+	DBCoreState()
+	  : logRouterTags(0), txsTags(0), recoveryCount(0), logSystemType(LogSystemType::empty),
+	    newestProtocolVersion(ProtocolVersion::invalidProtocolVersion),
+	    lowestCompatibleProtocolVersion(ProtocolVersion::invalidProtocolVersion) {}
 
 	std::vector<UID> getPriorCommittedLogServers() {
 		std::vector<UID> priorCommittedLogServers;
@@ -180,6 +185,9 @@ struct DBCoreState {
 			if (ar.protocolVersion().hasShardedTxsTags()) {
 				serializer(ar, txsTags);
 			}
+			if (ar.protocolVersion().hasSWVersionTracking()) {
+				serializer(ar, newestProtocolVersion, lowestCompatibleProtocolVersion);
+			}
 		} else if (ar.isDeserializing) {
 			tLogs.push_back(CoreTLogSet());
 			serializer(ar,

fdbserver/DDTeamCollection.actor.cpp

@@ -2039,6 +2039,8 @@ public:
 		}
 
 		loop {
+			state Future<Void> pauseChanged = self->pauseWiggle->onChange();
+			state Future<Void> stopChanged = stopSignal->onChange();
 			if (self->wigglingId.present()) {
 				state UID id = self->wigglingId.get();
 				if (self->pauseWiggle->get()) {
@@ -2067,7 +2069,7 @@ public:
 							    .detail("ExtraHealthyTeamCount", extraTeamCount)
 							    .detail("HealthyTeamCount", self->healthyTeamCount);
 						}
-						when(wait(self->pauseWiggle->onChange())) { continue; }
+						when(wait(pauseChanged)) { continue; }
 					}
 				}
 			}
@@ -2098,7 +2100,7 @@ public:
 					finishStorageWiggleSignal.send(Void());
 					extraTeamCount = std::max(0, extraTeamCount - 1);
 				}
-				when(wait(ddQueueCheck || self->pauseWiggle->onChange() || stopSignal->onChange())) {}
+				when(wait(ddQueueCheck || pauseChanged || stopChanged)) {}
 			}
 
 			if (stopSignal->get()) {
@@ -2631,7 +2633,11 @@ public:
 										    .detail("TSSID", tssId)
 										    .detail("Reason",
 										            self->zeroHealthyTeams->get() ? "ZeroHealthyTeams" : "TooMany");
+										Promise<Void> shutdown = self->shutdown;
 										killPromise.send(Void());
+										if (!shutdown.canBeSet()) {
+											return Void(); // "self" got destroyed, so return.
+										}
 									}
 								}
 							}
@@ -3582,6 +3588,8 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
 
 DDTeamCollection::~DDTeamCollection() {
 	TraceEvent("DDTeamCollectionDestructed", distributorId).detail("Primary", primary);
+	// Signal that the object is being destroyed.
+	shutdown.send(Void());
 
 	// Cancel the teamBuilder to avoid creating new teams after teams are cancelled.
 	teamBuilder.cancel();

fdbserver/DDTeamCollection.h

@@ -272,6 +272,12 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 
 	LocalityMap<UID> machineLocalityMap; // locality info of machines
 
+	// A mechanism to tell actors that reference a DDTeamCollection object through a direct
+	// pointer (without doing reference counting) that the object is being destroyed.
+	// (Introduced to solve the problem of "self" getting destroyed from underneath the
+	// "storageRecruiter" actor).
+	Promise<Void> shutdown;
+
 	// Randomly choose one machine team that has chosenServer and has the correct size
 	// When configuration is changed, we may have machine teams with old storageTeamSize
 	Reference<TCMachineTeamInfo> findOneRandomMachineTeam(TCServerInfo const& chosenServer) const;

fdbserver/DataDistributionQueue.actor.cpp

@@ -386,15 +386,19 @@ void launchDest(RelocateData& relocation,
 	}
 }
 
+void completeDest(RelocateData const& relocation, std::map<UID, Busyness>& destBusymap) {
+	int destWorkFactor = getDestWorkFactor();
+	for (UID id : relocation.completeDests) {
+		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
+	}
+}
+
 void complete(RelocateData const& relocation, std::map<UID, Busyness>& busymap, std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.workFactor > 0);
 	for (int i = 0; i < relocation.src.size(); i++)
 		busymap[relocation.src[i]].removeWork(relocation.priority, relocation.workFactor);
 
-	int destWorkFactor = getDestWorkFactor();
-	for (UID id : relocation.completeDests) {
-		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
-	}
+	completeDest(relocation, destBusymap);
 }
 
 ACTOR Future<Void> dataDistributionRelocator(struct DDQueueData* self,
@@ -1389,6 +1393,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			} else {
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
+
+				completeDest(rd, self->destBusymap);
 				rd.completeDests.clear();
 				wait(delay(SERVER_KNOBS->RETRY_RELOCATESHARD_DELAY, TaskPriority::DataDistributionLaunch));
 			}

fdbserver/EncryptKeyProxy.actor.cpp

@@ -21,11 +21,15 @@
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/Stats.h"
 #include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbserver/KmsConnector.h"
+#include "fdbserver/KmsConnectorInterface.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/SimEncryptKmsProxy.actor.h"
+#include "fdbserver/SimKmsConnector.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
 #include "flow/Error.h"
 #include "flow/EventTypes.actor.h"
 #include "flow/FastRef.h"
@@ -37,12 +41,11 @@
 #include "flow/network.h"
 
 #include <boost/mpl/not.hpp>
+#include <utility>
+#include <memory>
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-using EncryptDomainId = uint64_t;
-using EncryptBaseCipherId = uint64_t;
-
 namespace {
 bool canReplyWith(Error e) {
 	switch (e.code()) {
@@ -56,16 +59,16 @@ bool canReplyWith(Error e) {
 } // namespace
 
 struct EncryptBaseCipherKey {
-	EncryptDomainId domainId;
-	EncryptBaseCipherId baseCipherId;
+	EncryptCipherDomainId domainId;
+	EncryptCipherBaseKeyId baseCipherId;
 	Standalone<StringRef> baseCipherKey;
 	uint64_t creationTimeSec;
 	bool noExpiry;
 
 	EncryptBaseCipherKey()
 	  : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), creationTimeSec(0), noExpiry(false) {}
-	explicit EncryptBaseCipherKey(EncryptDomainId dId,
-	                              EncryptBaseCipherId cipherId,
+	explicit EncryptBaseCipherKey(EncryptCipherDomainId dId,
+	                              EncryptCipherBaseKeyId cipherId,
 	                              StringRef cipherKey,
 	                              bool neverExpire)
 	  : domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), creationTimeSec(now()), noExpiry(neverExpire) {
@@ -74,8 +77,8 @@ struct EncryptBaseCipherKey {
 	bool isValid() { return noExpiry ? true : ((now() - creationTimeSec) < FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL); }
 };
 
-using EncryptBaseDomainIdCache = std::unordered_map<EncryptDomainId, EncryptBaseCipherKey>;
-using EncryptBaseCipherKeyIdCache = std::unordered_map<EncryptBaseCipherId, EncryptBaseCipherKey>;
+using EncryptBaseDomainIdCache = std::unordered_map<EncryptCipherDomainId, EncryptBaseCipherKey>;
+using EncryptBaseCipherKeyIdCache = std::unordered_map<EncryptCipherBaseKeyId, EncryptBaseCipherKey>;
 
 struct EncryptKeyProxyData : NonCopyable, ReferenceCounted<EncryptKeyProxyData> {
 public:
@@ -86,6 +89,8 @@ public:
 	EncryptBaseDomainIdCache baseCipherDomainIdCache;
 	EncryptBaseCipherKeyIdCache baseCipherKeyIdCache;
 
+	std::unique_ptr<KmsConnector> kmsConnector;
+
 	CounterCollection ekpCacheMetrics;
 
 	Counter baseCipherKeyIdCacheMisses;
@@ -106,8 +111,8 @@ public:
 	    numResponseWithErrors("EKPNumResponseWithErrors", ekpCacheMetrics),
 	    numEncryptionKeyRefreshErrors("EKPNumEncryptionKeyRefreshErrors", ekpCacheMetrics) {}
 
-	void insertIntoBaseDomainIdCache(const EncryptDomainId domainId,
-	                                 const EncryptBaseCipherId baseCipherId,
+	void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
+	                                 const EncryptCipherBaseKeyId baseCipherId,
 	                                 const StringRef baseCipherKey) {
 		// Entries in domainId cache are eligible for periodic refreshes to support 'limiting lifetime of encryption
 		// key' support if enabled on external KMS solutions.
@@ -118,8 +123,8 @@ public:
 		insertIntoBaseCipherIdCache(domainId, baseCipherId, baseCipherKey);
 	}
 
-	void insertIntoBaseCipherIdCache(const EncryptDomainId domainId,
-	                                 const EncryptBaseCipherId baseCipherId,
+	void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId,
+	                                 const EncryptCipherBaseKeyId baseCipherId,
 	                                 const StringRef baseCipherKey) {
 		// Given an cipherKey is immutable, it is OK to NOT expire cached information.
 		// TODO: Update cache to support LRU eviction policy to limit the total cache size.
@@ -150,162 +155,168 @@ public:
 };
 
 ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData> ekpProxyData,
-                                                   SimKmsProxyInterface simKmsInterface,
+                                                   KmsConnectorInterface kmsConnectorInf,
                                                    EKPGetBaseCipherKeysByIdsRequest req) {
 	// Scan the cached cipher-keys and filter our baseCipherIds locally cached
 	// for the rest, reachout to KMS to fetch the required details
 
-	std::vector<EncryptBaseCipherId> lookupCipherIds;
-	state std::unordered_map<EncryptBaseCipherId, Standalone<StringRef>> cachedKeys;
-
-	for (EncryptBaseCipherId id : req.baseCipherIds) {
-		const auto itr = ekpProxyData->baseCipherKeyIdCache.find(id);
-		if (itr != ekpProxyData->baseCipherKeyIdCache.end()) {
-			ASSERT(itr->second.isValid());
-			cachedKeys.emplace(id, itr->second.baseCipherKey);
-		} else {
-			lookupCipherIds.push_back(id);
-		}
-	}
-
-	ekpProxyData->baseCipherKeyIdCacheHits += cachedKeys.size();
-	ekpProxyData->baseCipherKeyIdCacheMisses += lookupCipherIds.size();
+	std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> lookupCipherIds;
+	state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
 
 	state EKPGetBaseCipherKeysByIdsRequest keysByIds = req;
 	state EKPGetBaseCipherKeysByIdsReply keyIdsReply;
 
-	if (g_network->isSimulated()) {
-		if (!lookupCipherIds.empty()) {
-			try {
-				SimGetEncryptKeysByKeyIdsRequest simKeyIdsReq(lookupCipherIds);
-				SimGetEncryptKeysByKeyIdsReply simKeyIdsReply =
-				    wait(simKmsInterface.encryptKeyLookupByKeyIds.getReply(simKeyIdsReq));
+	// Dedup the requested pair<baseCipherId, encryptDomainId>
+	// TODO: endpoint serialization of std::unordered_set isn't working at the moment
+	std::unordered_set<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>,
+	                   boost::hash<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>>
+	    dedupedCipherIds;
+	for (const auto& item : req.baseCipherIds) {
+		dedupedCipherIds.emplace(item);
+	}
 
-				for (const auto& item : simKeyIdsReply.encryptKeyMap) {
-					keyIdsReply.baseCipherMap.emplace(item.first, StringRef(keyIdsReply.arena, item.second));
-				}
-
-				// Record the fetched cipher details to the local cache for the future references
-				// Note: cache warm-up is done after reponding to the caller
-
-				for (auto& item : simKeyIdsReply.encryptKeyMap) {
-					// DomainId isn't available here, the caller must know the encryption domainId
-					ekpProxyData->insertIntoBaseCipherIdCache(0, item.first, item.second);
-				}
-			} catch (Error& e) {
-				if (!canReplyWith(e)) {
-					TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e);
-					throw;
-				}
-				TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code());
-				ekpProxyData->sendErrorResponse(keysByIds.reply, e);
-				return Void();
-			}
+	for (const auto& item : dedupedCipherIds) {
+		const auto itr = ekpProxyData->baseCipherKeyIdCache.find(item.first);
+		if (itr != ekpProxyData->baseCipherKeyIdCache.end()) {
+			ASSERT(itr->second.isValid());
+			cachedCipherDetails.emplace_back(
+			    itr->second.domainId, itr->second.baseCipherId, itr->second.baseCipherKey, keyIdsReply.arena);
+		} else {
+			lookupCipherIds.emplace_back(std::make_pair(item.first, item.second));
 		}
-	} else {
-		// TODO: Call to non-FDB KMS connector process.
-		throw not_implemented();
 	}
 
-	for (auto& item : cachedKeys) {
-		keyIdsReply.baseCipherMap.emplace(item.first, item.second);
+	ekpProxyData->baseCipherKeyIdCacheHits += cachedCipherDetails.size();
+	ekpProxyData->baseCipherKeyIdCacheMisses += lookupCipherIds.size();
+
+	if (!lookupCipherIds.empty()) {
+		try {
+			KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds);
+			KmsConnLookupEKsByKeyIdsRep keysByIdsRep = wait(kmsConnectorInf.ekLookupByIds.getReply(keysByIdsReq));
+
+			for (const auto& item : keysByIdsRep.cipherKeyDetails) {
+				keyIdsReply.baseCipherDetails.emplace_back(
+				    item.encryptDomainId, item.encryptKeyId, item.encryptKey, keyIdsReply.arena);
+			}
+
+			// Record the fetched cipher details to the local cache for the future references
+			// Note: cache warm-up is done after reponding to the caller
+
+			for (auto& item : keysByIdsRep.cipherKeyDetails) {
+				// DomainId isn't available here, the caller must know the encryption domainId
+				ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
+			}
+		} catch (Error& e) {
+			if (!canReplyWith(e)) {
+				TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e);
+				throw;
+			}
+			TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code());
+			ekpProxyData->sendErrorResponse(keysByIds.reply, e);
+			return Void();
+		}
 	}
 
-	keyIdsReply.numHits = cachedKeys.size();
+	// Append cached cipherKeyDetails to the result-set
+	keyIdsReply.baseCipherDetails.insert(
+	    keyIdsReply.baseCipherDetails.end(), cachedCipherDetails.begin(), cachedCipherDetails.end());
+
+	keyIdsReply.numHits = cachedCipherDetails.size();
 	keysByIds.reply.send(keyIdsReply);
 
 	return Void();
 }
 
 ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyData,
-                                       SimKmsProxyInterface simKmsInterface,
+                                       KmsConnectorInterface kmsConnectorInf,
                                        EKPGetLatestBaseCipherKeysRequest req) {
 	// Scan the cached cipher-keys and filter our baseCipherIds locally cached
 	// for the rest, reachout to KMS to fetch the required details
 
-	state std::unordered_map<EncryptBaseCipherId, EKPBaseCipherDetails> cachedKeys;
+	state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
 	state EKPGetLatestBaseCipherKeysRequest latestKeysReq = req;
 	state EKPGetLatestBaseCipherKeysReply latestCipherReply;
 	state Arena& arena = latestCipherReply.arena;
 
+	// Dedup the requested domainIds.
+	// TODO: endpoint serialization of std::unordered_set isn't working at the moment
+	std::unordered_set<EncryptCipherDomainId> dedupedDomainIds;
+	for (EncryptCipherDomainId id : req.encryptDomainIds) {
+		dedupedDomainIds.emplace(id);
+	}
+
 	// First, check if the requested information is already cached by the server.
 	// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
 
-	std::vector<EncryptBaseCipherId> lookupCipherDomains;
-	for (EncryptDomainId id : req.encryptDomainIds) {
+	std::vector<EncryptCipherDomainId> lookupCipherDomains;
+	for (EncryptCipherDomainId id : dedupedDomainIds) {
 		const auto itr = ekpProxyData->baseCipherDomainIdCache.find(id);
 		if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
-			cachedKeys.emplace(id, EKPBaseCipherDetails(itr->second.baseCipherId, itr->second.baseCipherKey, arena));
+			cachedCipherDetails.emplace_back(id, itr->second.baseCipherId, itr->second.baseCipherKey, arena);
 		} else {
-			lookupCipherDomains.push_back(id);
+			lookupCipherDomains.emplace_back(id);
 		}
 	}
 
-	ekpProxyData->baseCipherDomainIdCacheHits += cachedKeys.size();
+	ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size();
 	ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size();
 
-	if (g_network->isSimulated()) {
-		if (!lookupCipherDomains.empty()) {
-			try {
-				SimGetEncryptKeysByDomainIdsRequest simKeysByDomainIdReq(lookupCipherDomains);
-				SimGetEncryptKeyByDomainIdReply simKeysByDomainIdRep =
-				    wait(simKmsInterface.encryptKeyLookupByDomainId.getReply(simKeysByDomainIdReq));
+	if (!lookupCipherDomains.empty()) {
+		try {
+			KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains);
+			KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep =
+			    wait(kmsConnectorInf.ekLookupByDomainIds.getReply(keysByDomainIdReq));
 
-				for (auto& item : simKeysByDomainIdRep.encryptKeyMap) {
-					latestCipherReply.baseCipherDetailMap.emplace(
-					    item.first, EKPBaseCipherDetails(item.second.encryptKeyId, item.second.encryptKey, arena));
+			for (auto& item : keysByDomainIdRep.cipherKeyDetails) {
+				latestCipherReply.baseCipherDetails.emplace_back(
+				    item.encryptDomainId, item.encryptKeyId, item.encryptKey, arena);
 
-					// Record the fetched cipher details to the local cache for the future references
-					ekpProxyData->insertIntoBaseDomainIdCache(
-					    item.first, item.second.encryptKeyId, item.second.encryptKey);
-				}
-			} catch (Error& e) {
-				if (!canReplyWith(e)) {
-					TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).error(e);
-					throw;
-				}
-				TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).detail("ErrorCode", e.code());
-				ekpProxyData->sendErrorResponse(latestKeysReq.reply, e);
-				return Void();
+				// Record the fetched cipher details to the local cache for the future references
+				ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
 			}
+		} catch (Error& e) {
+			if (!canReplyWith(e)) {
+				TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).error(e);
+				throw;
+			}
+			TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).detail("ErrorCode", e.code());
+			ekpProxyData->sendErrorResponse(latestKeysReq.reply, e);
+			return Void();
 		}
-	} else {
-		// TODO: Call to non-FDB KMS connector process.
-		throw not_implemented();
 	}
 
-	for (auto& item : cachedKeys) {
-		latestCipherReply.baseCipherDetailMap.emplace(
-		    item.first, EKPBaseCipherDetails(item.second.baseCipherId, item.second.baseCipherKey, arena));
+	for (auto& item : cachedCipherDetails) {
+		latestCipherReply.baseCipherDetails.emplace_back(
+		    item.encryptDomainId, item.baseCipherId, item.baseCipherKey, arena);
 	}
 
-	latestCipherReply.numHits = cachedKeys.size();
+	latestCipherReply.numHits = cachedCipherDetails.size();
 	latestKeysReq.reply.send(latestCipherReply);
 
 	return Void();
 }
 
-ACTOR Future<Void> refreshEncryptionKeysUsingSimKms(Reference<EncryptKeyProxyData> ekpProxyData,
-                                                    SimKmsProxyInterface simKmsInterface) {
+ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpProxyData,
+                                             KmsConnectorInterface kmsConnectorInf) {
 
 	ASSERT(g_network->isSimulated());
 
-	TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("Inf", simKmsInterface.id());
+	TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("KmsConnInf", kmsConnectorInf.id());
 
 	try {
-		SimGetEncryptKeysByDomainIdsRequest req;
+		KmsConnLookupEKsByDomainIdsReq req;
 		req.encryptDomainIds.reserve(ekpProxyData->baseCipherDomainIdCache.size());
 
 		for (auto& item : ekpProxyData->baseCipherDomainIdCache) {
 			req.encryptDomainIds.emplace_back(item.first);
 		}
-		SimGetEncryptKeyByDomainIdReply rep = wait(simKmsInterface.encryptKeyLookupByDomainId.getReply(req));
-		for (auto& item : rep.encryptKeyMap) {
-			ekpProxyData->insertIntoBaseDomainIdCache(item.first, item.second.encryptKeyId, item.second.encryptKey);
+		KmsConnLookupEKsByDomainIdsRep rep = wait(kmsConnectorInf.ekLookupByDomainIds.getReply(req));
+		for (auto& item : rep.cipherKeyDetails) {
+			ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
 		}
 
-		ekpProxyData->baseCipherKeysRefreshed += rep.encryptKeyMap.size();
-		TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.encryptKeyMap.size());
+		ekpProxyData->baseCipherKeysRefreshed += rep.cipherKeyDetails.size();
+		TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.cipherKeyDetails.size());
 	} catch (Error& e) {
 		if (!canReplyWith(e)) {
 			TraceEvent("RefreshEncryptionKeys_Error").error(e);
@@ -318,30 +329,34 @@ ACTOR Future<Void> refreshEncryptionKeysUsingSimKms(Reference<EncryptKeyProxyDat
 	return Void();
 }
 
-ACTOR Future<Void> refreshEncryptionKeysUsingKms(Reference<EncryptKeyProxyData> ekpProxyData) {
-	wait(delay(0)); // compiler needs to be happy
-	throw not_implemented();
+void refreshEncryptionKeys(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnectorInterface kmsConnectorInf) {
+	Future<Void> ignored = refreshEncryptionKeysCore(ekpProxyData, kmsConnectorInf);
 }
 
-void refreshEncryptionKeys(Reference<EncryptKeyProxyData> ekpProxyData, SimKmsProxyInterface simKmsInterface) {
-
-	Future<Void> ignored;
+void activateKmsConnector(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnectorInterface kmsConnectorInf) {
 	if (g_network->isSimulated()) {
-		ignored = refreshEncryptionKeysUsingSimKms(ekpProxyData, simKmsInterface);
+		ekpProxyData->kmsConnector = std::make_unique<SimKmsConnector>();
+	} else if (SERVER_KNOBS->KMS_CONNECTOR_TYPE.compare("HttpKmsConnector")) {
+		throw not_implemented();
 	} else {
-		ignored = refreshEncryptionKeysUsingKms(ekpProxyData);
+		throw not_implemented();
 	}
+
+	TraceEvent("EKP_ActiveKmsConnector", ekpProxyData->myId).detail("ConnectorType", SERVER_KNOBS->KMS_CONNECTOR_TYPE);
+	ekpProxyData->addActor.send(ekpProxyData->kmsConnector->connectorCore(kmsConnectorInf));
 }
 
 ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface, Reference<AsyncVar<ServerDBInfo>> db) {
 	state Reference<EncryptKeyProxyData> self(new EncryptKeyProxyData(ekpInterface.id()));
-	state PromiseStream<Future<Void>> addActor;
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 	self->addActor.send(traceRole(Role::ENCRYPT_KEY_PROXY, ekpInterface.id()));
 
-	state SimKmsProxyInterface simKmsProxyInf;
+	state KmsConnectorInterface kmsConnectorInf;
+	kmsConnectorInf.initEndpoints();
 
-	TraceEvent("EKP_Start", self->myId).log();
+	TraceEvent("EKP_Start", self->myId).detail("KmsConnectorInf", kmsConnectorInf.id());
+
+	activateKmsConnector(self, kmsConnectorInf);
 
 	// Register a recurring task to refresh the cached Encryption keys.
 	// Approach avoids external RPCs due to EncryptionKey refreshes for the inline write encryption codepath such as:
@@ -350,31 +365,17 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 	// FLOW_KNOB->ENCRRYPTION_KEY_REFRESH_INTERVAL_SEC, allowing the interactions with external Encryption Key Manager
 	// mostly not co-inciding with FDB process encryption key refresh attempts.
 
-	if (g_network->isSimulated()) {
-		// In simulation construct an Encryption KMSProxy actor to satisfy encryption keys lookups otherwise satisfied
-		// by integrating external Encryption Key Management solutions.
-
-		simKmsProxyInf.initEndpoints();
-		self->addActor.send(simEncryptKmsProxyCore(simKmsProxyInf));
-
-		TraceEvent("EKP_InitSimKmsInf", self->myId).detail("Inf", simKmsProxyInf.id());
-
-		self->encryptionKeyRefresher = recurring([&]() { refreshEncryptionKeys(self, simKmsProxyInf); },
-		                                         FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL,
-		                                         TaskPriority::Worker);
-
-	} else {
-		// TODO: Add recurring actor to talk to external KMS proxy process
-		throw not_implemented();
-	}
+	self->encryptionKeyRefresher = recurring([&]() { refreshEncryptionKeys(self, kmsConnectorInf); },
+	                                         FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL,
+	                                         TaskPriority::Worker);
 
 	try {
 		loop choose {
 			when(EKPGetBaseCipherKeysByIdsRequest req = waitNext(ekpInterface.getBaseCipherKeysByIds.getFuture())) {
-				wait(getCipherKeysByBaseCipherKeyIds(self, simKmsProxyInf, req));
+				wait(getCipherKeysByBaseCipherKeyIds(self, kmsConnectorInf, req));
 			}
 			when(EKPGetLatestBaseCipherKeysRequest req = waitNext(ekpInterface.getLatestBaseCipherKeys.getFuture())) {
-				wait(getLatestCipherKeys(self, simKmsProxyInf, req));
+				wait(getLatestCipherKeys(self, kmsConnectorInf, req));
 			}
 			when(HaltEncryptKeyProxyRequest req = waitNext(ekpInterface.haltEncryptKeyProxy.getFuture())) {
 				TraceEvent("EKP_Halted", self->myId).detail("ReqID", req.requesterID);

fdbserver/EncryptKeyProxyInterface.h

@@ -90,10 +90,26 @@ struct HaltEncryptKeyProxyRequest {
 	}
 };
 
+struct EKPBaseCipherDetails {
+	constexpr static FileIdentifier file_identifier = 2149615;
+	int64_t encryptDomainId;
+	uint64_t baseCipherId;
+	StringRef baseCipherKey;
+
+	EKPBaseCipherDetails() : encryptDomainId(0), baseCipherId(0), baseCipherKey(StringRef()) {}
+	explicit EKPBaseCipherDetails(int64_t dId, uint64_t id, StringRef key, Arena& arena)
+	  : encryptDomainId(dId), baseCipherId(id), baseCipherKey(StringRef(arena, key)) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptDomainId, baseCipherId, baseCipherKey);
+	}
+};
+
 struct EKPGetBaseCipherKeysByIdsReply {
 	constexpr static FileIdentifier file_identifier = 9485259;
 	Arena arena;
-	std::unordered_map<uint64_t, StringRef> baseCipherMap;
+	std::vector<EKPBaseCipherDetails> baseCipherDetails;
 	int numHits;
 	Optional<Error> error;
 
@@ -101,18 +117,18 @@ struct EKPGetBaseCipherKeysByIdsReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, baseCipherMap, numHits, error);
+		serializer(ar, arena, baseCipherDetails, numHits, error);
 	}
 };
 
 struct EKPGetBaseCipherKeysByIdsRequest {
 	constexpr static FileIdentifier file_identifier = 4930263;
 	UID requesterID;
-	std::vector<uint64_t> baseCipherIds;
+	std::vector<std::pair<uint64_t, int64_t>> baseCipherIds;
 	ReplyPromise<EKPGetBaseCipherKeysByIdsReply> reply;
 
 	EKPGetBaseCipherKeysByIdsRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
-	explicit EKPGetBaseCipherKeysByIdsRequest(UID uid, const std::vector<uint64_t>& ids)
+	explicit EKPGetBaseCipherKeysByIdsRequest(UID uid, const std::vector<std::pair<uint64_t, int64_t>>& ids)
 	  : requesterID(uid), baseCipherIds(ids) {}
 
 	template <class Ar>
@@ -121,35 +137,20 @@ struct EKPGetBaseCipherKeysByIdsRequest {
 	}
 };
 
-struct EKPBaseCipherDetails {
-	constexpr static FileIdentifier file_identifier = 2149615;
-	uint64_t baseCipherId;
-	StringRef baseCipherKey;
-
-	EKPBaseCipherDetails() : baseCipherId(0), baseCipherKey(StringRef()) {}
-	explicit EKPBaseCipherDetails(uint64_t id, StringRef key, Arena& arena)
-	  : baseCipherId(id), baseCipherKey(StringRef(arena, key)) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, baseCipherId, baseCipherKey);
-	}
-};
-
 struct EKPGetLatestBaseCipherKeysReply {
 	constexpr static FileIdentifier file_identifier = 4831583;
 	Arena arena;
-	std::unordered_map<uint64_t, EKPBaseCipherDetails> baseCipherDetailMap;
+	std::vector<EKPBaseCipherDetails> baseCipherDetails;
 	int numHits;
 	Optional<Error> error;
 
 	EKPGetLatestBaseCipherKeysReply() : numHits(0) {}
-	explicit EKPGetLatestBaseCipherKeysReply(const std::unordered_map<uint64_t, EKPBaseCipherDetails>& cipherMap)
-	  : baseCipherDetailMap(cipherMap), numHits(0) {}
+	explicit EKPGetLatestBaseCipherKeysReply(const std::vector<EKPBaseCipherDetails>& cipherDetails)
+	  : baseCipherDetails(cipherDetails), numHits(0) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, baseCipherDetailMap, numHits, error);
+		serializer(ar, arena, baseCipherDetails, numHits, error);
 	}
 };
 

fdbserver/FDBExecHelper.actor.cpp

@@ -242,7 +242,9 @@ ACTOR Future<int> spawnProcess(std::string binPath,
 
 static auto fork_child(const std::string& path, std::vector<char*>& paramList) {
 	int pipefd[2];
-	pipe(pipefd);
+	if (pipe(pipefd) != 0) {
+		return std::make_pair(-1, Optional<int>{});
+	}
 	auto readFD = pipefd[0];
 	auto writeFD = pipefd[1];
 	pid_t pid = fork();

fdbserver/GrvProxyServer.actor.cpp

@@ -542,7 +542,7 @@ ACTOR Future<Void> lastCommitUpdater(GrvProxyData* self, PromiseStream<Future<Vo
 	}
 }
 
-ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanID parentSpan,
+ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanContext parentSpan,
                                                           GrvProxyData* grvProxyData,
                                                           uint32_t flags,
                                                           Optional<UID> debugID,
@@ -581,8 +581,10 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanID parentSpan,
 	GetRawCommittedVersionReply repFromMaster = wait(replyFromMasterFuture);
 	grvProxyData->minKnownCommittedVersion =
 	    std::max(grvProxyData->minKnownCommittedVersion, repFromMaster.minKnownCommittedVersion);
-	// TODO add to "status json"
-	grvProxyData->ssVersionVectorCache.applyDelta(repFromMaster.ssVersionVectorDelta);
+	if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
+		// TODO add to "status json"
+		grvProxyData->ssVersionVectorCache.applyDelta(repFromMaster.ssVersionVectorDelta);
+	}
 	grvProxyData->stats.grvGetCommittedVersionRpcDist->sampleSeconds(now() - grvConfirmEpochLive);
 	GetReadVersionReply rep;
 	rep.version = repFromMaster.version;
@@ -646,8 +648,10 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		}
 		reply.midShardSize = midShardSize;
 		reply.tagThrottleInfo.clear();
-		grvProxyData->ssVersionVectorCache.getDelta(request.maxVersion, reply.ssVersionVectorDelta);
-		grvProxyData->versionVectorSizeOnGRVReply.addMeasurement(reply.ssVersionVectorDelta.size());
+		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
+			grvProxyData->ssVersionVectorCache.getDelta(request.maxVersion, reply.ssVersionVectorDelta);
+			grvProxyData->versionVectorSizeOnGRVReply.addMeasurement(reply.ssVersionVectorDelta.size());
+		}
 		reply.proxyId = grvProxyData->dbgid;
 
 		if (!request.tags.empty()) {
@@ -941,7 +945,7 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 		int batchGRVProcessed = 0;
 		for (int i = 0; i < start.size(); i++) {
 			if (start[i].size()) {
-				Future<GetReadVersionReply> readVersionReply = getLiveCommittedVersion(UID() /*span.context*/,
+				Future<GetReadVersionReply> readVersionReply = getLiveCommittedVersion(SpanContext(),
 				                                                                       grvProxyData,
 				                                                                       i,
 				                                                                       debugID,

fdbserver/IKeyValueStore.h

@@ -187,6 +187,8 @@ inline IKeyValueStore* openKVStore(KeyValueStoreType storeType,
 		return keyValueStoreRedwoodV1(filename, logID);
 	case KeyValueStoreType::SSD_ROCKSDB_V1:
 		return keyValueStoreRocksDB(filename, logID, storeType);
+	case KeyValueStoreType::SSD_SHARDED_ROCKSDB:
+		return keyValueStoreRocksDB(filename, logID, storeType); // TODO: to replace the KVS in the future
 	case KeyValueStoreType::MEMORY_RADIXTREE:
 		return keyValueStoreMemory(filename,
 		                           logID,

fdbserver/KeyValueStoreRocksDB.actor.cpp

@@ -50,6 +50,7 @@
 #endif
 #include "fdbclient/SystemData.h"
 #include "fdbserver/CoroFlow.h"
+#include "flow/ActorCollection.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
 #include "flow/ThreadHelper.actor.h"
@@ -863,16 +864,19 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		std::shared_ptr<ReadIteratorPool> readIterPool;
 		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
 		int threadIndex;
-		ThreadReturnPromiseStream<std::tuple<int, std::string, double>> metricPromiseStream;
+		ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream;
+		// ThreadReturnPromiseStream pair.first stores the histogram name and
+		// pair.second stores the corresponding measured latency (seconds)
 
 		explicit Writer(DB& db,
 		                CF& cf,
 		                UID id,
 		                std::shared_ptr<ReadIteratorPool> readIterPool,
 		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
-		                int threadIndex)
+		                int threadIndex,
+		                ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream)
 		  : db(db), cf(cf), id(id), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
-		    threadIndex(threadIndex),
+		    threadIndex(threadIndex), metricPromiseStream(metricPromiseStream),
 		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
 		                    ? rocksdb::NewGenericRateLimiter(
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
@@ -1041,8 +1045,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double commitBeginTime;
 			if (a.getHistograms) {
 				commitBeginTime = timer_monotonic();
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString(), commitBeginTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString(), commitBeginTime - a.startTime));
 			}
 			Standalone<VectorRef<KeyRangeRef>> deletes;
 			DeleteVisitor dv(deletes, deletes.arena());
@@ -1066,8 +1070,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			s = db->Write(options, a.batchToCommit.get());
 			readIterPool->update();
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_WRITE_HISTOGRAM.toString(), timer_monotonic() - writeBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_WRITE_HISTOGRAM.toString(), timer_monotonic() - writeBeginTime));
 			}
 
 			if (!s.ok()) {
@@ -1083,17 +1087,16 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					ASSERT(db->SuggestCompactRange(cf, &begin, &end).ok());
 				}
 				if (a.getHistograms) {
-					metricPromiseStream.send(std::make_tuple(threadIndex,
-					                                         ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString(),
+					metricPromiseStream->send(std::make_pair(ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString(),
 					                                         timer_monotonic() - compactRangeBeginTime));
 				}
 			}
 			if (a.getHistograms) {
 				double currTime = timer_monotonic();
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString(), currTime - commitBeginTime));
-				metricPromiseStream.send(
-				    std::make_tuple(threadIndex, ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString(), currTime - commitBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
 			}
 			if (doPerfContextMetrics) {
 				perfContextMetrics->set(threadIndex);
@@ -1267,15 +1270,18 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		std::shared_ptr<ReadIteratorPool> readIterPool;
 		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
 		int threadIndex;
-		ThreadReturnPromiseStream<std::tuple<int, std::string, double>> metricPromiseStream;
+		ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream;
+		// ThreadReturnPromiseStream pair.first stores the histogram name and
+		// pair.second stores the corresponding measured latency (seconds)
 
 		explicit Reader(DB& db,
 		                CF& cf,
 		                std::shared_ptr<ReadIteratorPool> readIterPool,
 		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
-		                int threadIndex)
+		                int threadIndex,
+		                ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream)
 		  : db(db), cf(cf), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
-		    threadIndex(threadIndex) {
+		    metricPromiseStream(metricPromiseStream), threadIndex(threadIndex) {
 			if (g_network->isSimulated()) {
 				// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
 				// very high load and single read thread cannot process all the load within the timeouts.
@@ -1319,8 +1325,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
 			}
 			Optional<TraceBatch> traceBatch;
 			if (a.debugID.present()) {
@@ -1352,8 +1358,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READVALUE_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READVALUE_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
 			}
 
 			if (a.debugID.present()) {
@@ -1371,10 +1377,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			if (a.getHistograms) {
 				double currTime = timer_monotonic();
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READVALUE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READVALUE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READVALUE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READVALUE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
 			}
 			if (doPerfContextMetrics) {
 				perfContextMetrics->set(threadIndex);
@@ -1404,8 +1410,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
 			}
 			Optional<TraceBatch> traceBatch;
 			if (a.debugID.present()) {
@@ -1433,8 +1439,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
 			auto s = db->Get(options, cf, toSlice(a.key), &value);
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READPREFIX_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READPREFIX_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
 			}
 
 			if (a.debugID.present()) {
@@ -1454,10 +1460,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 			if (a.getHistograms) {
 				double currTime = timer_monotonic();
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READPREFIX_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READPREFIX_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READPREFIX_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READPREFIX_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
 			}
 			if (doPerfContextMetrics) {
 				perfContextMetrics->set(threadIndex);
@@ -1486,8 +1492,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
 			}
 			if (readBeginTime - a.startTime > readRangeTimeout) {
 				TraceEvent(SevWarn, "KVSTimeout")
@@ -1508,8 +1514,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
 				ReadIterator readIter = readIterPool->getIterator();
 				if (a.getHistograms) {
-					metricPromiseStream.send(std::make_tuple(threadIndex,
-					                                         ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
+					metricPromiseStream->send(std::make_pair(ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
 					                                         timer_monotonic() - iterCreationBeginTime));
 				}
 				auto cursor = readIter.iter;
@@ -1538,8 +1543,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
 				ReadIterator readIter = readIterPool->getIterator();
 				if (a.getHistograms) {
-					metricPromiseStream.send(std::make_tuple(threadIndex,
-					                                         ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
+					metricPromiseStream->send(std::make_pair(ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
 					                                         timer_monotonic() - iterCreationBeginTime));
 				}
 				auto cursor = readIter.iter;
@@ -1582,10 +1586,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			a.result.send(result);
 			if (a.getHistograms) {
 				double currTime = timer_monotonic();
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
-				metricPromiseStream.send(std::make_tuple(
-				    threadIndex, ROCKSDB_READRANGE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
+				metricPromiseStream->send(
+				    std::make_pair(ROCKSDB_READRANGE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
 			}
 			if (doPerfContextMetrics) {
 				perfContextMetrics->set(threadIndex);
@@ -1611,7 +1615,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	FlowLock fetchSemaphore;
 	int numFetchWaiters;
 	std::shared_ptr<ReadIteratorPool> readIterPool;
-	std::vector<Future<Void>> actors;
+	std::vector<std::unique_ptr<ThreadReturnPromiseStream<std::pair<std::string, double>>>> metricPromiseStreams;
+	// ThreadReturnPromiseStream pair.first stores the histogram name and
+	// pair.second stores the corresponding measured latency (seconds)
+	Future<Void> actorErrorListener;
+	Future<Void> collection;
+	PromiseStream<Future<Void>> addActor;
 
 	struct Counters {
 		CounterCollection cc;
@@ -1649,23 +1658,52 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			writeThread = createGenericThreadPool();
 			readThreads = createGenericThreadPool();
 		}
-		struct Writer* writer =
-		    new Writer(db, defaultFdbCF, id, readIterPool, perfContextMetrics, SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
 		if (SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0) {
-			actors.push_back(updateHistogram(writer->metricPromiseStream.getFuture()));
+			collection = actorCollection(addActor.getFuture());
+			for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM + 1; i++) {
+				// ROCKSDB_READ_PARALLELISM readers and 1 writer
+				metricPromiseStreams.emplace_back(
+				    std::make_unique<ThreadReturnPromiseStream<std::pair<std::string, double>>>());
+				addActor.send(updateHistogram(metricPromiseStreams[i]->getFuture()));
+			}
 		}
-		writeThread->addThread(writer, "fdb-rocksdb-wr");
+
+		// the writer uses SERVER_KNOBS->ROCKSDB_READ_PARALLELISM as its threadIndex
+		// threadIndex is used for metricPromiseStreams and perfContextMetrics
+		writeThread->addThread(new Writer(db,
+		                                  defaultFdbCF,
+		                                  id,
+		                                  readIterPool,
+		                                  perfContextMetrics,
+		                                  SERVER_KNOBS->ROCKSDB_READ_PARALLELISM,
+		                                  SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0
+		                                      ? metricPromiseStreams[SERVER_KNOBS->ROCKSDB_READ_PARALLELISM].get()
+		                                      : nullptr),
+		                       "fdb-rocksdb-wr");
 		TraceEvent("RocksDBReadThreads").detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
 		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
-			struct Reader* reader = new Reader(db, defaultFdbCF, readIterPool, perfContextMetrics, i);
-			if (SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0) {
-				actors.push_back(updateHistogram(reader->metricPromiseStream.getFuture()));
-			}
-			readThreads->addThread(reader, "fdb-rocksdb-re");
+			readThreads->addThread(
+			    new Reader(db,
+			               defaultFdbCF,
+			               readIterPool,
+			               perfContextMetrics,
+			               i,
+			               SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0 ? metricPromiseStreams[i].get() : nullptr),
+			    "fdb-rocksdb-re");
 		}
 	}
 
-	ACTOR Future<Void> updateHistogram(FutureStream<std::tuple<int, std::string, double>> metricFutureStream) {
+	ACTOR Future<Void> errorListenActor(Future<Void> collection) {
+		try {
+			wait(collection);
+			ASSERT(false);
+			throw internal_error();
+		} catch (Error& e) {
+			throw e;
+		}
+	}
+
+	ACTOR Future<Void> updateHistogram(FutureStream<std::pair<std::string, double>> metricFutureStream) {
 		state Reference<Histogram> commitLatencyHistogram = Histogram::getHistogram(
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_LATENCY_HISTOGRAM, Histogram::Unit::microseconds);
 		state Reference<Histogram> commitActionHistogram = Histogram::getHistogram(
@@ -1702,9 +1740,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::microseconds);
 		loop {
 			choose {
-				when(std::tuple<int, std::string, double> measure = waitNext(metricFutureStream)) {
-					std::string metricName = std::get<1>(measure);
-					double latency = std::get<2>(measure);
+				when(std::pair<std::string, double> measure = waitNext(metricFutureStream)) {
+					std::string metricName = measure.first;
+					double latency = measure.second;
 					if (metricName == ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString()) {
 						commitLatencyHistogram->sampleSeconds(latency);
 					} else if (metricName == ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString()) {
@@ -1739,6 +1777,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 						readValueGetHistogram->sampleSeconds(latency);
 					} else if (metricName == ROCKSDB_READPREFIX_GET_HISTOGRAM.toString()) {
 						readPrefixGetHistogram->sampleSeconds(latency);
+					} else {
+						UNREACHABLE();
 					}
 				}
 			}
@@ -1769,7 +1809,14 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	void close() override { doClose(this, false); }
 
-	KeyValueStoreType getType() const override { return KeyValueStoreType(KeyValueStoreType::SSD_ROCKSDB_V1); }
+	KeyValueStoreType getType() const override {
+		if (SERVER_KNOBS->ENABLE_SHARDED_ROCKSDB)
+			// KVSRocks pretends as KVSShardedRocksDB
+			// TODO: to remove when the ShardedRocksDB KVS implementation is added in the future
+			return KeyValueStoreType(KeyValueStoreType::SSD_SHARDED_ROCKSDB);
+		else
+			return KeyValueStoreType(KeyValueStoreType::SSD_ROCKSDB_V1);
+	}
 
 	Future<Void> init() override {
 		if (openFuture.isValid()) {
@@ -1807,11 +1854,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	ACTOR Future<Void> checkRocksdbState(RocksDBKeyValueStore* self) {
 		state uint64_t estPendCompactBytes;
 		state int count = SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
-		self->db->GetIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
+		self->db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes,
+		                                   &estPendCompactBytes);
 		while (count && estPendCompactBytes > SERVER_KNOBS->ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT) {
 			wait(delay(SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD));
 			count--;
-			self->db->GetIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
+			self->db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes,
+			                                   &estPendCompactBytes);
 		}
 
 		return Void();

fdbserver/KmsConnector.h

@@ -0,0 +1,43 @@
+/*
+ * KmsConnector.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef KMS_CONNECTOR_H
+#define KMS_CONNECTOR_H
+#pragma once
+
+#include "fdbserver/KmsConnectorInterface.h"
+#include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
+
+// FDB encryption needs to interact with external Key Management Services (KMS) solutions to lookup/refresh encryption
+// keys. KmsConnector interface is an abstract interface enabling implementing specialized KMS connector
+// implementations.
+// FDB KMSConnector implementation should inherit from KmsConnector and implement pure virtual function,
+// EncryptKeyProxyServer instantiates desired implementation object based on SERVER_KNOB->KMS_CONNECTOR_TYPE knob.
+
+class KmsConnector : public NonCopyable {
+public:
+	KmsConnector() {}
+	virtual ~KmsConnector() {}
+
+	virtual Future<Void> connectorCore(struct KmsConnectorInterface interf) = 0;
+};
+
+#endif

fdbserver/KmsConnectorInterface.h

@@ -0,0 +1,144 @@
+/*
+ * KmsConnectorInterface.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_KMSCONNECTORINTERFACE_H
+#define FDBSERVER_KMSCONNECTORINTERFACE_H
+#pragma once
+
+#include "fdbrpc/fdbrpc.h"
+#include "flow/EncryptUtils.h"
+#include "flow/FileIdentifier.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+
+struct KmsConnectorInterface {
+	constexpr static FileIdentifier file_identifier = 2416711;
+	RequestStream<ReplyPromise<Void>> waitFailure;
+	RequestStream<struct KmsConnLookupEKsByKeyIdsReq> ekLookupByIds;
+	RequestStream<struct KmsConnLookupEKsByDomainIdsReq> ekLookupByDomainIds;
+
+	KmsConnectorInterface() {}
+
+	UID id() const { return ekLookupByIds.getEndpoint().token; }
+	template <class Archive>
+	void serialize(Archive& ar) {
+		if constexpr (!is_fb_function<Archive>) {
+			ASSERT(ar.protocolVersion().isValid());
+		}
+		serializer(ar, waitFailure);
+		if (Archive::isDeserializing) {
+			ekLookupByIds =
+			    RequestStream<struct KmsConnLookupEKsByKeyIdsReq>(waitFailure.getEndpoint().getAdjustedEndpoint(1));
+			ekLookupByDomainIds =
+			    RequestStream<struct KmsConnLookupEKsByDomainIdsReq>(waitFailure.getEndpoint().getAdjustedEndpoint(2));
+		}
+	}
+
+	void initEndpoints() {
+		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
+		streams.push_back(waitFailure.getReceiver());
+		streams.push_back(ekLookupByIds.getReceiver(TaskPriority::Worker));
+		streams.push_back(ekLookupByDomainIds.getReceiver(TaskPriority::Worker));
+		FlowTransport::transport().addEndpoints(streams);
+	}
+};
+
+struct EncryptCipherKeyDetails {
+	constexpr static FileIdentifier file_identifier = 1227025;
+	EncryptCipherDomainId encryptDomainId;
+	EncryptCipherBaseKeyId encryptKeyId;
+	StringRef encryptKey;
+
+	EncryptCipherKeyDetails() {}
+	explicit EncryptCipherKeyDetails(EncryptCipherDomainId dId,
+	                                 EncryptCipherBaseKeyId keyId,
+	                                 StringRef key,
+	                                 Arena& arena)
+	  : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(StringRef(arena, key)) {}
+
+	bool operator==(const EncryptCipherKeyDetails& toCompare) {
+		return encryptDomainId == toCompare.encryptDomainId && encryptKeyId == toCompare.encryptKeyId &&
+		       encryptKey.compare(toCompare.encryptKey) == 0;
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptDomainId, encryptKeyId, encryptKey);
+	}
+};
+
+struct KmsConnLookupEKsByKeyIdsRep {
+	constexpr static FileIdentifier file_identifier = 2313778;
+	Arena arena;
+	std::vector<EncryptCipherKeyDetails> cipherKeyDetails;
+
+	KmsConnLookupEKsByKeyIdsRep() {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, arena, cipherKeyDetails);
+	}
+};
+
+struct KmsConnLookupEKsByKeyIdsReq {
+	constexpr static FileIdentifier file_identifier = 6913396;
+	std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> encryptKeyIds;
+	ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
+
+	KmsConnLookupEKsByKeyIdsReq() {}
+	explicit KmsConnLookupEKsByKeyIdsReq(
+	    const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds)
+	  : encryptKeyIds(keyIds) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptKeyIds, reply);
+	}
+};
+
+struct KmsConnLookupEKsByDomainIdsRep {
+	constexpr static FileIdentifier file_identifier = 3009025;
+	Arena arena;
+	std::vector<EncryptCipherKeyDetails> cipherKeyDetails;
+
+	KmsConnLookupEKsByDomainIdsRep() {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, arena, cipherKeyDetails);
+	}
+};
+
+struct KmsConnLookupEKsByDomainIdsReq {
+	constexpr static FileIdentifier file_identifier = 9918682;
+	std::vector<EncryptCipherDomainId> encryptDomainIds;
+	ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
+
+	KmsConnLookupEKsByDomainIdsReq() {}
+	explicit KmsConnLookupEKsByDomainIdsReq(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptDomainIds, reply);
+	}
+};
+
+#endif

fdbserver/LeaderElection.actor.cpp

@@ -51,7 +51,7 @@ ACTOR Future<Void> submitCandidacy(Key key,
 				    .detail("OldAddr", coord.candidacy.getEndpoint().getPrimaryAddress().toString());
 				if (rep.getError().code() == error_code_request_maybe_delivered) {
 					// Delay to prevent tight resolving loop due to outdated DNS cache
-					wait(delay(FLOW_KNOBS->HOSTNAME_RESOLVE_DELAY));
+					wait(delay(FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL));
 					throw coordinators_changed();
 				} else {
 					throw rep.getError();

fdbserver/LogRouter.actor.cpp

@@ -18,21 +18,24 @@
  * limitations under the License.
  */
 
-#include "flow/ActorCollection.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbclient/SystemData.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/RecoveryState.h"
 #include "fdbclient/Atomic.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbrpc/Stats.h"
+#include "fdbserver/ApplyMetadataMutation.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/LogSystem.h"
+#include "fdbserver/WaitFailure.h"
+#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/RecoveryState.h"
+#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/TLogInterface.h"
+#include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Histogram.h"
 #include "flow/TDMetric.actor.h"
+#include "flow/network.h"
+
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct LogRouterData {
@@ -526,6 +529,10 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 		    .detail("Begin", reqBegin)
 		    .detail("Popped", poppedVer)
 		    .detail("Start", self->startVersion);
+		if (std::is_same<PromiseType, Promise<TLogPeekReply>>::value) {
+			// kills logRouterPeekStream actor, otherwise that actor becomes stuck
+			throw operation_obsolete();
+		}
 		replyPromise.send(Never());
 		if (reqSequence.present()) {
 			auto& trackerData = self->peekTracker[peekId];

fdbserver/LogSystem.cpp

@@ -19,6 +19,9 @@
  */
 
 #include "fdbserver/LogSystem.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbserver/OTELSpanContextMessage.h"
+#include "fdbserver/SpanContextMessage.h"
 #include "flow/serialize.h"
 
 std::string LogSet::logRouterString() {
@@ -277,8 +280,8 @@ void LogPushData::addTxsTag() {
 	}
 }
 
-void LogPushData::addTransactionInfo(SpanID const& context) {
-	TEST(!spanContext.isValid()); // addTransactionInfo with invalid SpanID
+void LogPushData::addTransactionInfo(SpanContext const& context) {
+	TEST(!spanContext.isValid()); // addTransactionInfo with invalid SpanContext
 	spanContext = context;
 	writtenLocations.clear();
 }
@@ -344,13 +347,33 @@ bool LogPushData::writeTransactionInfo(int location, uint32_t subseq) {
 	writtenLocations.insert(location);
 
 	BinaryWriter& wr = messagesWriter[location];
-	SpanContextMessage contextMessage(spanContext);
-
 	int offset = wr.getLength();
 	wr << uint32_t(0) << subseq << uint16_t(prev_tags.size());
 	for (auto& tag : prev_tags)
 		wr << tag;
-	wr << contextMessage;
+	if (logSystem->getTLogVersion() >= TLogVersion::V7) {
+		OTELSpanContextMessage contextMessage(spanContext);
+		wr << contextMessage;
+	} else {
+		// When we're on a TLog version below 7, but the front end of the system (i.e. proxy, sequencer, resolver)
+		// is using OpenTelemetry tracing (i.e on or above 7.2), we need to convert the OpenTelemetry Span data model
+		// i.e. 16 bytes for traceId, 8 bytes for spanId, to the OpenTracing spec, which is 8 bytes for traceId
+		// and 8 bytes for spanId. That means we need to drop some data.
+		//
+		// As a workaround for this special case we've decided to drop is the 8 bytes
+		// for spanId. Therefore we're passing along the full 16 byte traceId to the storage server with 0 for spanID.
+		// This will result in a follows from relationship for the storage span within the trace rather than a
+		// parent->child.
+		SpanContextMessage contextMessage;
+		if (spanContext.isSampled()) {
+			TEST(true); // Converting OTELSpanContextMessage to traced SpanContextMessage
+			contextMessage = SpanContextMessage(UID(spanContext.traceID.first(), spanContext.traceID.second()));
+		} else {
+			TEST(true); // Converting OTELSpanContextMessage to untraced SpanContextMessage
+			contextMessage = SpanContextMessage(UID(0, 0));
+		}
+		wr << contextMessage;
+	}
 	int length = wr.getLength() - offset;
 	*(uint32_t*)((uint8_t*)wr.getData() + offset) = length - sizeof(uint32_t);
 	return true;

fdbserver/LogSystem.h

@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "fdbserver/SpanContextMessage.h"
+#include "fdbserver/OTELSpanContextMessage.h"
 #include "fdbserver/TLogInterface.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbclient/DatabaseConfiguration.h"
@@ -519,7 +520,7 @@ struct ILogSystem {
 	                             Version knownCommittedVersion,
 	                             Version minKnownCommittedVersion,
 	                             LogPushData& data,
-	                             SpanID const& spanContext,
+	                             SpanContext const& spanContext,
 	                             Optional<UID> debugID = Optional<UID>(),
 	                             Optional<std::unordered_map<uint16_t, Version>> tpcvMap =
 	                                 Optional<std::unordered_map<uint16_t, Version>>()) = 0;
@@ -762,7 +763,7 @@ struct LogPushData : NonCopyable {
 	}
 
 	// Add transaction info to be written before the first mutation in the transaction.
-	void addTransactionInfo(SpanID const& context);
+	void addTransactionInfo(SpanContext const& context);
 
 	// copy written_tags, after filtering, into given set
 	void saveTags(std::set<Tag>& filteredTags) const {
@@ -832,7 +833,7 @@ private:
 	// field.
 	std::unordered_set<int> writtenLocations;
 	uint32_t subsequence;
-	SpanID spanContext;
+	SpanContext spanContext;
 	bool shardChanged = false; // if keyServers has any changes, i.e., shard boundary modifications.
 
 	// Writes transaction info to the message stream at the given location if

fdbserver/MasterInterface.h

@@ -133,14 +133,14 @@ struct GetCommitVersionReply {
 
 struct GetCommitVersionRequest {
 	constexpr static FileIdentifier file_identifier = 16683181;
-	SpanID spanContext;
+	SpanContext spanContext;
 	uint64_t requestNum;
 	uint64_t mostRecentProcessedRequestNum;
 	UID requestingProxy;
 	ReplyPromise<GetCommitVersionReply> reply;
 
 	GetCommitVersionRequest() {}
-	GetCommitVersionRequest(SpanID spanContext,
+	GetCommitVersionRequest(SpanContext spanContext,
 	                        uint64_t requestNum,
 	                        uint64_t mostRecentProcessedRequestNum,
 	                        UID requestingProxy)