Merge branch 'master' of github.com:apple/foundationdb into add-coordinators-into-special-keys

CMakeLists.txt

@@ -72,7 +72,8 @@ add_custom_target(branch_file ALL DEPENDS ${CURR_BRANCH_FILE})
 execute_process(
   COMMAND git rev-parse HEAD
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-  OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL)
+  OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
 string(STRIP "${CURRENT_GIT_VERSION_WNL}" CURRENT_GIT_VERSION)
 message(STATUS "Current git version ${CURRENT_GIT_VERSION}")
 
@@ -182,6 +183,12 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
   add_link_options(-lexecinfo)
 endif()
 
+################################################################################
+# Build information
+################################################################################
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/fdbclient/BuildFlags.h)
+
 ################################################################################
 # process compile commands for IDE
 ################################################################################

FDBLibTLS/FDBLibTLSPlugin.cpp

@@ -38,7 +38,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() {
 	if (rc < 0) {
 		// Log the failure from tls_init during our constructor.
 		TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls");
-		return NULL;
+		return nullptr;
 	}
 	return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this));
 }
@@ -47,5 +47,5 @@ extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name
 	if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) {
 		return new FDBLibTLSPlugin;
 	}
-	return NULL;
+	return nullptr;
 }

FDBLibTLS/FDBLibTLSPolicy.cpp

@@ -37,11 +37,11 @@
 #include <string.h>
 #include <limits.h>
 
-FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin):
+FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin)
-	plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false),
+  : plugin(plugin), tls_cfg(nullptr), roots(nullptr), session_created(false), ca_data_set(false), cert_data_set(false),
-	cert_data_set(false), key_data_set(false), verify_peers_set(false) {
+    key_data_set(false), verify_peers_set(false) {
 
-	if ((tls_cfg = tls_config_new()) == NULL) {
+	if ((tls_cfg = tls_config_new()) == nullptr) {
 		TraceEvent(SevError, "FDBLibTLSConfigError");
 		throw std::runtime_error("FDBLibTLSConfigError");
 	}
@@ -55,29 +55,31 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
 	tls_config_free(tls_cfg);
 }
 
-ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
+ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func,
+                                             void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
 	if (is_client) {
 		// If verify peers has been set then there is no point specifying a
 		// servername, since this will be ignored - the servername should be
 		// matched by the verify criteria instead.
-		if (verify_peers_set && servername != NULL) {
+		if (verify_peers_set && servername != nullptr) {
 			TraceEvent(SevError, "FDBLibTLSVerifyPeersWithServerName");
-			return NULL;
+			return nullptr;
 		}
 
 		// If verify peers has not been set, then require a server name to
 		// avoid an accidental lack of name validation.
-		if (!verify_peers_set && servername == NULL) {
+		if (!verify_peers_set && servername == nullptr) {
 			TraceEvent(SevError, "FDBLibTLSNoServerName");
-			return NULL;
+			return nullptr;
 		}
 	}
 
 	session_created = true;
 	try {
-		return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func, send_ctx, recv_func, recv_ctx, uid);
+		return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func,
+		                            send_ctx, recv_func, recv_ctx, uid);
 	} catch ( ... ) {
-		return NULL;
+		return nullptr;
 	}
 }
 
@@ -87,8 +89,7 @@ static int password_cb(char *buf, int size, int rwflag, void *u) {
 
 	if (size < 0)
 		return 0;
-	if (u == NULL)
+	if (u == nullptr) return 0;
-		return 0;
 
 	plen = strlen(password);
 	if (plen > size)
@@ -102,24 +103,24 @@ static int password_cb(char *buf, int size, int rwflag, void *u) {
 }
 
 struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len) {
-	struct stack_st_X509 *certs = NULL;
+	struct stack_st_X509* certs = nullptr;
-	X509 *cert = NULL;
+	X509* cert = nullptr;
-	BIO *bio = NULL;
+	BIO* bio = nullptr;
 	int errnum;
 
 	if (cert_pem_len > INT_MAX)
 		goto err;
-	if ((bio = BIO_new_mem_buf((void *)cert_pem, cert_pem_len)) == NULL) {
+	if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == nullptr) {
 		TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 		goto err;
 	}
-	if ((certs = sk_X509_new_null()) == NULL) {
+	if ((certs = sk_X509_new_null()) == nullptr) {
 		TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 		goto err;
 	}
 
 	ERR_clear_error();
-	while ((cert = PEM_read_bio_X509(bio, NULL, password_cb, NULL)) != NULL) {
+	while ((cert = PEM_read_bio_X509(bio, nullptr, password_cb, nullptr)) != nullptr) {
 		if (!sk_X509_push(certs, cert)) {
 			TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 			goto err;
@@ -150,7 +151,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
 	X509_free(cert);
 	BIO_free(bio);
 
-	return NULL;
+	return nullptr;
 }
 
 bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
@@ -166,8 +167,7 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
 	if (ca_len < 0)
 		return false;
 	sk_X509_pop_free(roots, X509_free);
-	if ((roots = parse_cert_pem(ca_data, ca_len)) == NULL)
+	if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr) return false;
-		return false;
 
 	if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) {
 		TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
@@ -200,8 +200,8 @@ bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) {
 }
 
 bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const char* password) {
-	EVP_PKEY *key = NULL;
+	EVP_PKEY* key = nullptr;
-	BIO *bio = NULL;
+	BIO* bio = nullptr;
 	bool rc = false;
 
 	if (key_data_set) {
@@ -213,16 +213,16 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
 		goto err;
 	}
 
-	if (password != NULL) {
+	if (password != nullptr) {
 		char *data;
 		long len;
 
-		if ((bio = BIO_new_mem_buf((void *)key_data, key_len)) == NULL) {
+		if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) {
 			TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 			goto err;
 		}
 		ERR_clear_error();
-		if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void *)password)) == NULL) {
+		if ((key = PEM_read_bio_PrivateKey(bio, nullptr, password_cb, (void*)password)) == nullptr) {
 			int errnum = ERR_peek_error();
 			char errbuf[256];
 
@@ -236,11 +236,11 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
 			goto err;
 		}
 		BIO_free(bio);
-		if ((bio = BIO_new(BIO_s_mem())) == NULL) {
+		if ((bio = BIO_new(BIO_s_mem())) == nullptr) {
 			TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 			goto err;
 		}
-		if (!PEM_write_bio_PrivateKey(bio, key, NULL, NULL, 0, NULL, NULL)) {
+		if (!PEM_write_bio_PrivateKey(bio, key, nullptr, nullptr, 0, nullptr, nullptr)) {
 			TraceEvent(SevError, "FDBLibTLSOutOfMemory");
 			goto err;
 		}

FDBLibTLS/FDBLibTLSSession.cpp

@@ -60,14 +60,16 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
 	return (ssize_t)rv;
 }
 
-FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uidptr) :
+FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername,
-	tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
+                                   TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func,
-	recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
+                                   void* recv_ctx, void* uidptr)
+  : tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
+    recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
 	if (uidptr)
 		uid = * (UID*) uidptr;
 
 	if (is_client) {
-		if ((tls_ctx = tls_client()) == NULL) {
+		if ((tls_ctx = tls_client()) == nullptr) {
 			TraceEvent(SevError, "FDBLibTLSClientError", uid);
 			throw std::runtime_error("FDBLibTLSClientError");
 		}
@@ -82,7 +84,7 @@ FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_cl
 			throw std::runtime_error("FDBLibTLSConnectError");
 		}
 	} else {
-		if ((tls_sctx = tls_server()) == NULL) {
+		if ((tls_sctx = tls_server()) == nullptr) {
 			TraceEvent(SevError, "FDBLibTLSServerError", uid);
 			throw std::runtime_error("FDBLibTLSServerError");
 		}
@@ -108,14 +110,13 @@ FDBLibTLSSession::~FDBLibTLSSession() {
 
 bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, MatchType mt) {
 	bool rc = false;
-	ASN1_STRING* asn_criteria = NULL;
+	ASN1_STRING* asn_criteria = nullptr;
-	unsigned char* criteria_utf8 = NULL;
+	unsigned char* criteria_utf8 = nullptr;
 	int criteria_utf8_len = 0;
-	unsigned char* entry_utf8 = NULL;
+	unsigned char* entry_utf8 = nullptr;
 	int entry_utf8_len = 0;
 
-	if ((asn_criteria = ASN1_IA5STRING_new()) == NULL)
+	if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr) goto err;
-		goto err;
 	if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1)
 		goto err;
 	if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1)
@@ -152,8 +153,7 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
 		return false;
 	if (X509_NAME_get_index_by_NID(name, nid, idx) != -1)
 		return false;
-	if ((name_entry = X509_NAME_get_entry(name, idx)) == NULL)
+	if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr) return false;
-		return false;
 
 	return match_criteria_entry(criteria, name_entry->value, mt);
 }
@@ -169,8 +169,9 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
 	}
 	std::string value_gen = value.substr(0, pos);
 	std::string value_val = value.substr(pos+1, value.npos);
-	STACK_OF(GENERAL_NAME)* sans = reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, NULL, NULL));
+	STACK_OF(GENERAL_NAME)* sans =
-	if (sans == NULL) {
+	    reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, nullptr, nullptr));
+	if (sans == nullptr) {
 		return false;
 	}
 	int num_sans = sk_GENERAL_NAME_num( sans );
@@ -231,10 +232,10 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
 }
 
 std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
-	X509_STORE_CTX *store_ctx = NULL;
+	X509_STORE_CTX* store_ctx = nullptr;
 	X509_NAME *subject, *issuer;
 	bool rc = false;
-	X509* cert = NULL;
+	X509* cert = nullptr;
 	// if returning false, give a reason string
 	std::string reason = "";
 
@@ -243,12 +244,12 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 		return std::make_tuple(true, reason);
 
 	// Verify the certificate.
-	if ((store_ctx = X509_STORE_CTX_new()) == NULL) {
+	if ((store_ctx = X509_STORE_CTX_new()) == nullptr) {
 		TraceEvent(SevError, "FDBLibTLSOutOfMemory", uid);
 		reason = "Out of memory";
 		goto err;
 	}
-	if (!X509_STORE_CTX_init(store_ctx, NULL, sk_X509_value(certs, 0), certs)) {
+	if (!X509_STORE_CTX_init(store_ctx, nullptr, sk_X509_value(certs, 0), certs)) {
 		reason = "Store ctx init";
 		goto err;
 	}
@@ -264,7 +265,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 
 	// Check subject criteria.
 	cert = sk_X509_value(store_ctx->chain, 0);
-	if ((subject = X509_get_subject_name(cert)) == NULL) {
+	if ((subject = X509_get_subject_name(cert)) == nullptr) {
 		reason = "Cert subject error";
 		goto err;
 	}
@@ -276,7 +277,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 	}
 
 	// Check issuer criteria.
-	if ((issuer = X509_get_issuer_name(cert)) == NULL) {
+	if ((issuer = X509_get_issuer_name(cert)) == nullptr) {
 		reason = "Cert issuer error";
 		goto err;
 	}
@@ -289,7 +290,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 
 	// Check root criteria - this is the subject of the final certificate in the stack.
 	cert = sk_X509_value(store_ctx->chain, sk_X509_num(store_ctx->chain) - 1);
-	if ((subject = X509_get_subject_name(cert)) == NULL) {
+	if ((subject = X509_get_subject_name(cert)) == nullptr) {
 		reason = "Root subject error";
 		goto err;
 	}
@@ -310,7 +311,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 }
 
 bool FDBLibTLSSession::verify_peer() {
-	struct stack_st_X509 *certs = NULL;
+	struct stack_st_X509* certs = nullptr;
 	const uint8_t *cert_pem;
 	size_t cert_pem_len;
 	bool rc = false;
@@ -323,12 +324,11 @@ bool FDBLibTLSSession::verify_peer() {
 	if (policy->verify_rules.empty())
 		return true;
 
-	if ((cert_pem = tls_peer_cert_chain_pem(tls_ctx, &cert_pem_len)) == NULL) {
+	if ((cert_pem = tls_peer_cert_chain_pem(tls_ctx, &cert_pem_len)) == nullptr) {
 		TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
 		goto err;
 	}
-	if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == NULL)
+	if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr) goto err;
-		goto err;
 
 	// Any matching rule is sufficient.
 	for (auto &verify_rule: policy->verify_rules) {

FDBLibTLS/FDBLibTLSVerify.cpp

@@ -147,7 +147,7 @@ static NID abbrevToNID(std::string const& sn) {
 
 static X509Location locationForNID(NID nid) {
 	const char* name = OBJ_nid2ln(nid);
-	if (name == NULL) {
+	if (name == nullptr) {
 		throw std::runtime_error("locationForNID");
 	}
 	if (strncmp(name, "X509v3", 6) == 0) {

bindings/c/CMakeLists.txt

@@ -111,6 +111,14 @@ if(NOT WIN32)
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
   target_link_libraries(mako PRIVATE fdb_c)
 
+  if(NOT OPEN_FOR_IDE)
+    # Make sure that fdb_c.h is compatible with c90
+    add_executable(fdb_c90_test test/fdb_c90_test.c)
+    set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
+    target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
+    target_link_libraries(fdb_c90_test PRIVATE fdb_c)
+  endif()
+
   add_fdbclient_test(
     NAME fdb_c_setup_tests
     COMMAND $<TARGET_FILE:fdb_c_setup_tests>)

bindings/c/fdb_c.cpp

@@ -389,6 +389,14 @@ fdb_error_t fdb_database_create_transaction( FDBDatabase* d,
 		*out_transaction = (FDBTransaction*)tr.extractPtr(); );
 }
 
+extern "C" DLLEXPORT FDBFuture* fdb_database_reboot_worker(FDBDatabase* db, uint8_t const* address, int address_length,
+                                                           fdb_bool_t check, int duration) {
+	return (FDBFuture*)(DB(db)->rebootWorker(StringRef(address, address_length), check, duration).extractPtr());
+}
+
+extern "C" DLLEXPORT FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* db, uint8_t const* dcid, int dcid_length) {
+	return (FDBFuture*)(DB(db)->forceRecoveryWithDataLoss(StringRef(dcid, dcid_length)).extractPtr());
+}
 
 extern "C" DLLEXPORT
 void fdb_transaction_destroy( FDBTransaction* tr ) {

bindings/c/foundationdb/fdb_c.h

@@ -45,12 +45,17 @@
 #define WARN_UNUSED_RESULT
 #endif
 
-// With default settings, gcc will not warn about unprototyped functions being called, so it
+/*
-// is easy to erroneously call a function which is not available at FDB_API_VERSION and then
+ * With default settings, gcc will not warn about unprototyped functions being
-// get an error only at runtime.  These macros ensure a compile error in such cases, and
+ * called, so it is easy to erroneously call a function which is not available
-// attempt to make the compile error slightly informative.
+ * at FDB_API_VERSION and then get an error only at runtime.  These macros
-#define This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION() [=====]
+ * ensure a compile error in such cases, and attempt to make the compile error
-#define FDB_REMOVED_FUNCTION This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION(0)
+ * slightly informative.
+ */
+#define This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION()    \
+  [== == = ]
+#define FDB_REMOVED_FUNCTION                                                   \
+  This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION(0)
 
 #include <stdint.h>
 
@@ -173,6 +178,13 @@ extern "C" {
     fdb_database_create_transaction( FDBDatabase* d,
                                      FDBTransaction** out_transaction );
 
+    DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
+    fdb_database_reboot_worker( FDBDatabase* db, uint8_t const* address,
+                                int address_length, fdb_bool_t check, int duration);
+    
+    DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
+    fdb_database_force_recovery_with_data_loss( FDBDatabase* db, uint8_t const* dcid, int dcid_length);
+
     DLLEXPORT void fdb_transaction_destroy( FDBTransaction* tr);
 
     DLLEXPORT void fdb_transaction_cancel( FDBTransaction* tr);
@@ -244,12 +256,15 @@ extern "C" {
     fdb_transaction_get_committed_version( FDBTransaction* tr,
                                            int64_t* out_version );
 
-    // This function intentionally returns an FDBFuture instead of an integer directly,
+    /*
-    // so that calling this API can see the effect of previous mutations on the transaction.
+     * This function intentionally returns an FDBFuture instead of an integer
-    // Specifically, mutations are applied asynchronously by the main thread. In order to
+     * directly, so that calling this API can see the effect of previous
-    // see them, this call has to be serviced by the main thread too.
+     * mutations on the transaction. Specifically, mutations are applied
-    DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
+     * asynchronously by the main thread. In order to see them, this call has to
-    fdb_transaction_get_approximate_size(FDBTransaction* tr);
+     * be serviced by the main thread too.
+     */
+    DLLEXPORT WARN_UNUSED_RESULT FDBFuture *
+    fdb_transaction_get_approximate_size(FDBTransaction *tr);
 
     DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
     fdb_get_server_protocol(const char* clusterFilePath);
@@ -301,7 +316,7 @@ extern "C" {
     typedef struct FDB_cluster FDBCluster;
 
     typedef enum {
-        // This option is only a placeholder for C compatibility and should not be used
+        /* This option is only a placeholder for C compatibility and should not be used */
         FDB_CLUSTER_OPTION_DUMMY_DO_NOT_USE=-1
     } FDBClusterOption;
 #endif

bindings/c/test/fdb_c90_test.c

@@ -0,0 +1,9 @@
+#define FDB_API_VERSION 700
+#include <foundationdb/fdb_c.h>
+
+int main(int argc, char* argv[]) {
+	(void)argc;
+	(void)argv;
+	fdb_select_api_version(700);
+	return 0;
+}

bindings/c/test/unit/fdb_api.cpp

@@ -92,6 +92,16 @@ void Future::cancel() {
   return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
 }
 
+// Database
+Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
+                                   int duration) {
+	return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
+}
+
+EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8_t *dcid, int dcid_length) {
+  return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
+}
+
 // Transaction
 
 Transaction::Transaction(FDBDatabase* db) {

bindings/c/test/unit/fdb_api.hpp

@@ -77,7 +77,6 @@ class Future {
   FDBFuture* future_;
 };
 
-
 class Int64Future : public Future {
  public:
    // Call this function instead of fdb_future_get_int64 when using the
@@ -86,6 +85,7 @@ class Int64Future : public Future {
   
  private:
   friend class Transaction;
+  friend class Database;
   Int64Future(FDBFuture* f) : Future(f) {}
 };
 
@@ -144,9 +144,18 @@ class KeyValueArrayFuture : public Future {
 class EmptyFuture : public Future {
  private:
   friend class Transaction;
+  friend class Database;
   EmptyFuture(FDBFuture* f) : Future(f) {}
 };
 
+// Wrapper around FDBDatabase, providing database-level API
+class Database final {
+public:
+	static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
+	                                int duration);
+  static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
+};
+
 // Wrapper around FDBTransaction, providing the same set of calls as the C API.
 // Handles cleanup of memory, removing the need to call
 // fdb_transaction_destroy.

bindings/c/test/unit/unit_tests.cpp

@@ -37,6 +37,7 @@
 
 #define DOCTEST_CONFIG_IMPLEMENT
 #include "doctest.h"
+#include "fdbclient/rapidjson/document.h"
 
 #include "fdb_api.hpp"
 
@@ -1881,19 +1882,19 @@ TEST_CASE("special-key-space disable tracing") {
   }
 }
 
-TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE") {
+TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE") {
-  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
+  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
 
   auto value = get_value("\xff\xff/tracing/token", /* snapshot */ false, {});
   REQUIRE(value.has_value());
   uint64_t token = std::stoul(value.value());
   CHECK(token == 0);
 
-  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
+  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
 }
 
-TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
+TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
-  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
+  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
 
   fdb::Transaction tr(db);
   fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES,
@@ -1921,7 +1922,7 @@ TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transactio
     break;
   }
 
-  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
+  fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
 }
 
 TEST_CASE("special-key-space tracing get range") {
@@ -1967,6 +1968,84 @@ TEST_CASE("special-key-space tracing get range") {
   }
 }
 
+std::string get_valid_status_json() {
+  fdb::Transaction tr(db);
+  while (1) {
+    fdb::ValueFuture f1 = tr.get("\xff\xff/status/json", false);
+    fdb_error_t err = wait_future(f1);
+    if (err) {
+      fdb::EmptyFuture f2 = tr.on_error(err);
+      fdb_check(wait_future(f2));
+      continue;
+    }
+
+    int out_present;
+    char *val;
+    int vallen;
+    fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen));
+    assert(out_present);
+    std::string statusJsonStr(val, vallen);
+    rapidjson::Document statusJson;
+    statusJson.Parse(statusJsonStr.c_str());
+    // make sure it is available
+    bool available = statusJson["client"]["database_status"]["available"].GetBool();
+    if (!available)
+      continue; // cannot reach to the cluster, retry
+    return statusJsonStr;
+  }
+}
+
+TEST_CASE("fdb_database_reboot_worker") {
+  std::string status_json = get_valid_status_json();
+  rapidjson::Document statusJson;
+  statusJson.Parse(status_json.c_str());
+  CHECK(statusJson.HasMember("cluster"));
+  CHECK(statusJson["cluster"].HasMember("generation"));
+  int old_generation = statusJson["cluster"]["generation"].GetInt();
+  CHECK(statusJson["cluster"].HasMember("processes"));
+  // Make sure we only have one process in the cluster
+  // Thus, rebooting the worker ensures a recovery
+  // Configuration changes may break the contract here
+  CHECK(statusJson["cluster"]["processes"].MemberCount() == 1);
+  auto processPtr = statusJson["cluster"]["processes"].MemberBegin();
+  CHECK(processPtr->value.HasMember("address"));
+  std::string network_address = processPtr->value["address"].GetString();
+  while (1) {
+	  fdb::Int64Future f =
+		  fdb::Database::reboot_worker(db, (const uint8_t*)network_address.c_str(), network_address.size(), false, 0);
+	  fdb_check(wait_future(f));
+	  int64_t successful;
+	  fdb_check(f.get(&successful));
+	  if (successful) break; // retry rebooting until success
+  }
+  status_json = get_valid_status_json();
+  statusJson.Parse(status_json.c_str());
+  CHECK(statusJson.HasMember("cluster"));
+  CHECK(statusJson["cluster"].HasMember("generation"));
+  int new_generation = statusJson["cluster"]["generation"].GetInt();
+  // The generation number should increase after the recovery
+  CHECK(new_generation > old_generation);
+}
+
+TEST_CASE("fdb_database_force_recovery_with_data_loss") {
+	// This command cannot be tested completely in the current unit test configuration
+	// For now, we simply call the function to make sure it exist
+	// Background:
+	// It is also only usable when usable_regions=2, so it requires a fearless configuration
+	// In particular, you have two data centers, and the storage servers in one region are allowed to fall behind (async
+	// replication) Normally, you would not want to recover to that set of storage servers unless there are tlogs which
+	// can let those storage servers catch up However, if all the tlogs are dead and you still want to be able to
+	// recover your database even if that means losing recently committed mutation, that's the time this function works
+
+	std::string dcid = "test_id";
+	while (1) {
+		fdb::EmptyFuture f =
+		    fdb::Database::force_recovery_with_data_loss(db, (const uint8_t*)dcid.c_str(), dcid.size());
+		fdb_check(wait_future(f));
+		break;
+  }
+}
+
 TEST_CASE("fdb_error_predicate") {
   CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1007)); // transaction_too_old
   CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1020)); // not_committed

bindings/flow/fdb_flow.actor.cpp

@@ -20,6 +20,7 @@
 
 #include "fdb_flow.h"
 
+#include <cstdint>
 #include <stdio.h>
 #include <cinttypes>
 
@@ -101,6 +102,8 @@ namespace FDB {
 
 		Reference<Transaction> createTransaction() override;
 		void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
+		Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
+		Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 
 	private:
 		FDBDatabase* db;
@@ -284,6 +287,23 @@ namespace FDB {
 			throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
 	}
 
+	Future<int64_t> DatabaseImpl::rebootWorker(const StringRef &address, bool check, int duration) {
+		return backToFuture<int64_t>( fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), [](Reference<CFuture> f) {
+				int64_t res;
+
+				throw_on_error(fdb_future_get_int64( f->f, &res ) );
+
+				return res;
+			} );
+	}
+
+	Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef &dcid) {
+		return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){
+			throw_on_error( fdb_future_get_error( f->f ) );
+			return Void();
+		});
+	}
+
 	TransactionImpl::TransactionImpl(FDBDatabase* db) {
 		throw_on_error(fdb_database_create_transaction(db, &tr));
 	}

bindings/flow/fdb_flow.h

@@ -124,6 +124,8 @@ namespace FDB {
 		virtual ~Database(){};
 		virtual Reference<Transaction> createTransaction() = 0;
 		virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
+		virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
+		virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
 	};
 
 	class API {

build/Dockerfile

@@ -1,17 +1,27 @@
 FROM centos:6
 
+# Clean yum cache, disable default Base repo and enable Vault
+RUN yum clean all &&\
+  sed -i -e 's/gpgcheck=1/enabled=0/g' /etc/yum.repos.d/CentOS-Base.repo &&\
+  sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo &&\
+  sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo &&\
+  sed -i -e "s/6\.0/$(cut -d\  -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo &&\
+  yum install -y yum-utils &&\
+  yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
+  yum -y install centos-release-scl-rh epel-release  \
+    http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
+  sed -i -e 's/#baseurl=/baseurl=/g' -e 's/mirror.centos.org/vault.centos.org/g' \
+     -e 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo &&\
+  yum clean all
+
 # Install dependencies for developer tools, bindings,\
 # documentation, actorcompiler, and packaging tools\
-RUN yum install -y yum-utils &&\
+RUN yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
-  yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
+      devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
-  yum -y install centos-release-scl epel-release \
+      devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
-    http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
+      rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
-  yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
+      mono-core debbuild python-pip dos2unix valgrind-devel ccache \
-    devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
+      distcc wget libxslt git lz4 lz4-devel lz4-static &&\
-    devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
-    rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
-    mono-core debbuild python-pip dos2unix valgrind-devel ccache \
-    distcc wget git lz4 lz4-devel lz4-static &&\
   pip install boto3==1.1.1
 
 USER root
@@ -41,6 +51,8 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1
 
 # install Ninja
 RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\
+    echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef  ninja.zip" > /tmp/ninja-sha.txt &&\
+    sha256sum -c /tmp/ninja-sha.txt &&\
     unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\
     cd .. && rm -rf ninja-1.9.0 ninja.zip
 
@@ -53,17 +65,59 @@ RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o o
     ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\
     cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz
 
+# Install toml11
+ RUN cd /tmp && curl -L https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz > toml.tar.gz &&\
+     echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d  toml.tar.gz" > toml-sha256.txt &&\
+     sha256sum -c toml-sha256.txt &&\
+     tar xf toml.tar.gz && rm -rf build && mkdir build && cd build && scl enable devtoolset-8 -- cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml11-3.4.0 &&\
+     scl enable devtoolset-8 -- cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/toml11-3.4.0
+
 RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\
     echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee  rocksdb.tar.gz" > rocksdb-sha.txt &&\
     sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt
 
+RUN cd /opt/ && curl -L https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o ctest2junit.xsl
+
+# Setting this environment variable switches from OpenSSL to BoringSSL
+#ENV OPENSSL_ROOT_DIR=/opt/boringssl
+
+# install BoringSSL:  TODO: They don't seem to have releases(?)  I picked today's master SHA.
+RUN cd /opt &&\
+    git clone https://boringssl.googlesource.com/boringssl &&\
+    cd boringssl &&\
+    git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\
+    mkdir build
+
+# ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS
+# also, enable -fPIC.
+# this is moderately uglier than creating a patchfile, but easier to maintain.
+RUN cd /opt/boringssl &&\
+	for f in crypto/fipsmodule/rand/fork_detect_test.cc \
+	         include/openssl/bn.h \
+		 ssl/test/bssl_shim.cc ; do \
+		perl -p -i -e 's/#include <inttypes.h>/#define __STDC_FORMAT_MACROS 1\n#include <inttypes.h>/g;' $f ; \
+	done &&\
+	perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\
+	git diff
+
+RUN cd /opt/boringssl/build &&\
+    scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\
+    scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\
+	./ssl/ssl_test &&\
+	mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib
+
 # Localize time zone
 ARG TIMEZONEINFO=America/Los_Angeles
 RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime
 
-LABEL version=0.1.19
+LABEL version=0.1.22
-ENV DOCKER_IMAGEVER=0.1.19
+ENV DOCKER_IMAGEVER=0.1.22
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
+
+ENV CCACHE_NOHASHDIR=true
+ENV CCACHE_UMASK=0000
+ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
+
 CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash

build/Dockerfile.devel

@@ -1,4 +1,5 @@
-FROM foundationdb/foundationdb-build:0.1.19
+ARG IMAGE_TAG=0.1.21
+FROM foundationdb/foundationdb-build:${IMAGE_TAG}
 
 USER root
 
@@ -50,8 +51,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\
 	ldconfig &&\
 	rm -rf /mnt/artifacts
 
-LABEL version=0.11.10
+LABEL version=0.11.13
-ENV DOCKER_IMAGEVER=0.11.10
+ENV DOCKER_IMAGEVER=0.11.13
 
 ENV CLANGCC=/usr/local/bin/clang.de8a65ef
 ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef
@@ -63,8 +64,5 @@ ENV CC=/usr/local/bin/clang.de8a65ef
 ENV CXX=/usr/local/bin/clang++.de8a65ef
 ENV USE_LD=LLD
 ENV USE_LIBCXX=1
-ENV CCACHE_NOHASHDIR=true
-ENV CCACHE_UMASK=0000
-ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
 
 CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash

build/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   common: &common
-    image: foundationdb/foundationdb-build:0.1.19
+    image: foundationdb/foundationdb-build:0.1.22
 
   build-setup: &build-setup
     <<: *common

cmake/AddFdbTest.cmake

@@ -383,6 +383,9 @@ function(add_fdbclient_test)
   set(oneValueArgs NAME)
   set(multiValueArgs COMMAND)
   cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
+  if(OPEN_FOR_IDE)
+    return()
+  endif()
   if(NOT T_ENABLED AND T_DISABLED)
     return()
   endif()

cmake/ConfigureCompiler.cmake

@@ -273,7 +273,6 @@ else()
       -Wno-unused-function
       -Wno-unused-local-typedef
       -Wno-unused-parameter
-      -Wno-unused-value
       -Wno-self-assign
       )
     if (USE_CCACHE)
@@ -312,9 +311,9 @@ else()
   endif()
 
   if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
-    # Graviton or later
+    # Graviton2 or later
     # https://github.com/aws/aws-graviton-gettting-started
-    add_compile_options(-march=armv8-a+crc+simd)
+    add_compile_options(-march=armv8.2-a+crc+simd)
   endif()
 
   # Check whether we can use dtrace probes

cmake/InstallLayout.cmake

@@ -303,15 +303,15 @@ set(CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME "foundationdb-clients")
 set(CPACK_RPM_SERVER-EL7_PACKAGE_NAME "foundationdb-server")
 set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_NAME "foundationdb-server-${PROJECT_VERSION}")
 
-set(CPACK_RPM_CLIENTS-EL7_FILE_NAME "${rpm-clients-filename}.el7.x86_64.rpm")
+set(CPACK_RPM_CLIENTS-EL7_FILE_NAME "${rpm-clients-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-filename}.versioned.x86_64.rpm")
+set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-EL7_FILE_NAME "${rpm-server-filename}.el7.x86_64.rpm")
+set(CPACK_RPM_SERVER-EL7_FILE_NAME "${rpm-server-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-filename}.versioned.x86_64.rpm")
+set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 
-set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.el7-debuginfo.x86_64.rpm")
+set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.versioned-debuginfo.x86_64.rpm")
+set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-EL7_DEBUGINFO_FILE_NAME "${rpm-server-filename}.el7-debuginfo.x86_64.rpm")
+set(CPACK_RPM_SERVER-EL7_DEBUGINFO_FILE_NAME "${rpm-server-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-filename}.versioned-debuginfo.x86_64.rpm")
+set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 
 file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir")
 fdb_install(DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir/" DESTINATION data COMPONENT server)
@@ -377,8 +377,13 @@ set(CPACK_RPM_CLIENTS-VERSIONED_PRE_UNINSTALL_SCRIPT_FILE
 # Configuration for DEB
 ################################################################################
 
-set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb")
+  set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb")
+  set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb")
+else()
+  set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
+  set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
+endif()
 set(CPACK_DEB_COMPONENT_INSTALL ON)
 set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ${GENERATE_DEBUG_PACKAGES})
 set(CPACK_DEBIAN_PACKAGE_SECTION "database")
@@ -427,8 +432,8 @@ endif()
 ################################################################################
 
 set(CPACK_ARCHIVE_COMPONENT_INSTALL ON)
-set(CPACK_ARCHIVE_CLIENTS-TGZ_FILE_NAME "${deb-clients-filename}.x86_64")
+set(CPACK_ARCHIVE_CLIENTS-TGZ_FILE_NAME "${deb-clients-filename}.${CMAKE_SYSTEM_PROCESSOR}")
-set(CPACK_ARCHIVE_SERVER-TGZ_FILE_NAME "${deb-server-filename}.x86_64")
+set(CPACK_ARCHIVE_SERVER-TGZ_FILE_NAME "${deb-server-filename}.${CMAKE_SYSTEM_PROCESSOR}")
 
 ################################################################################
 # Server configuration

documentation/sphinx/source/api-c.rst

@@ -426,6 +426,40 @@ An |database-blurb1| Modifications to a database are performed via transactions.
    ``*out_transaction``
       Set to point to the newly created :type:`FDBTransaction`.
 
+.. function:: FDBFuture* fdb_database_reboot_worker(FDBDatabase* database, uint8_t const* address, int address_length, fdb_bool_t check, int duration)
+
+   Reboot the specified process in the database.
+
+   |future-return0| a :type:`int64_t` which represents whether the reboot request is sent or not. In particular, 1 means request sent and 0 means failure (e.g. the process with the specified address does not exist). |future-return1| call :func:`fdb_future_get_int64()` to extract the result, |future-return2|
+
+   ``address``
+        A pointer to the network address of the process.
+
+   ``address_length``
+        |length-of| ``address``.
+   
+   ``check``
+        whether to perform a storage engine integrity check. In particular, the check-on-reboot is implemented by writing a check/validation file on disk as breadcrumb for the process to find after reboot, at which point it will eat the breadcrumb file and pass true to the integrityCheck parameter of the openKVStore() factory method.
+   
+   ``duration``
+        If positive, the process will be first suspended for ``duration`` seconds before being rebooted.
+
+.. function:: FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* database, uint8_t const* dcId, int dcId_length)
+
+   Force the database to recover into the given datacenter.
+
+   This function is only useful in a fearless configuration where you want to recover your database even with losing recently committed mutations.
+   
+   In particular, the function will set usable_regions to 1 and the amount of mutations that will be lost depends on how far behind the remote datacenter is.
+   
+   The function will change the region configuration to have a positive priority for the chosen dcId, and a negative priority for all other dcIds.
+
+   In particular, no error will be thrown if the given dcId does not exist. It will just not attemp to force a recovery.
+   
+   If the database has already recovered, the function does nothing. Thus it's safe to call it multiple times.
+
+   |future-returnvoid|
+
 Transaction
 ===========
 

documentation/sphinx/source/command-line-interface.rst

@@ -474,6 +474,11 @@ Prints a list of currently active transaction tag throttles, or recommended tran
 
 ``LIMIT`` - The number of throttles to print. Defaults to 100.
 
+triggerddteaminfolog
+--------------------
+
+The ``triggerddteaminfolog`` command would trigger the data distributor to log very detailed teams information into trace event logs.
+
 unlock
 ------
 

documentation/sphinx/source/downloads.rst

@@ -10,38 +10,38 @@ macOS
 
 The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
 
-* `FoundationDB-6.3.9.pkg <https://www.foundationdb.org/downloads/6.3.9/macOS/installers/FoundationDB-6.3.9.pkg>`_
+* `FoundationDB-6.3.10.pkg <https://www.foundationdb.org/downloads/6.3.10/macOS/installers/FoundationDB-6.3.10.pkg>`_
 
 Ubuntu
 ------
 
 The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
 
-* `foundationdb-clients-6.3.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.9/ubuntu/installers/foundationdb-clients_6.3.9-1_amd64.deb>`_
+* `foundationdb-clients-6.3.10-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.10/ubuntu/installers/foundationdb-clients_6.3.10-1_amd64.deb>`_
-* `foundationdb-server-6.3.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.9/ubuntu/installers/foundationdb-server_6.3.9-1_amd64.deb>`_ (depends on the clients package)
+* `foundationdb-server-6.3.10-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.10/ubuntu/installers/foundationdb-server_6.3.10-1_amd64.deb>`_ (depends on the clients package)
 
 RHEL/CentOS EL6
 ---------------
 
 The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
 
-* `foundationdb-clients-6.3.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-clients-6.3.9-1.el6.x86_64.rpm>`_
+* `foundationdb-clients-6.3.10-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.10/rhel6/installers/foundationdb-clients-6.3.10-1.el6.x86_64.rpm>`_
-* `foundationdb-server-6.3.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-server-6.3.9-1.el6.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-server-6.3.10-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.10/rhel6/installers/foundationdb-server-6.3.10-1.el6.x86_64.rpm>`_ (depends on the clients package)
 
 RHEL/CentOS EL7
 ---------------
 
 The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
 
-* `foundationdb-clients-6.3.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-clients-6.3.9-1.el7.x86_64.rpm>`_
+* `foundationdb-clients-6.3.10-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.10/rhel7/installers/foundationdb-clients-6.3.10-1.el7.x86_64.rpm>`_
-* `foundationdb-server-6.3.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-server-6.3.9-1.el7.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-server-6.3.10-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.10/rhel7/installers/foundationdb-server-6.3.10-1.el7.x86_64.rpm>`_ (depends on the clients package)
 
 Windows
 -------
 
 The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
 
-* `foundationdb-6.3.9-x64.msi <https://www.foundationdb.org/downloads/6.3.9/windows/installers/foundationdb-6.3.9-x64.msi>`_
+* `foundationdb-6.3.10-x64.msi <https://www.foundationdb.org/downloads/6.3.10/windows/installers/foundationdb-6.3.10-x64.msi>`_
 
 API Language Bindings
 =====================
@@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
 
 If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package:
 
-* `foundationdb-6.3.9.tar.gz <https://www.foundationdb.org/downloads/6.3.9/bindings/python/foundationdb-6.3.9.tar.gz>`_
+* `foundationdb-6.3.10.tar.gz <https://www.foundationdb.org/downloads/6.3.10/bindings/python/foundationdb-6.3.10.tar.gz>`_
 
 Ruby 1.9.3/2.0.0+
 -----------------
 
-* `fdb-6.3.9.gem <https://www.foundationdb.org/downloads/6.3.9/bindings/ruby/fdb-6.3.9.gem>`_
+* `fdb-6.3.10.gem <https://www.foundationdb.org/downloads/6.3.10/bindings/ruby/fdb-6.3.10.gem>`_
 
 Java 8+
 -------
 
-* `fdb-java-6.3.9.jar <https://www.foundationdb.org/downloads/6.3.9/bindings/java/fdb-java-6.3.9.jar>`_
+* `fdb-java-6.3.10.jar <https://www.foundationdb.org/downloads/6.3.10/bindings/java/fdb-java-6.3.10.jar>`_
-* `fdb-java-6.3.9-javadoc.jar <https://www.foundationdb.org/downloads/6.3.9/bindings/java/fdb-java-6.3.9-javadoc.jar>`_
+* `fdb-java-6.3.10-javadoc.jar <https://www.foundationdb.org/downloads/6.3.10/bindings/java/fdb-java-6.3.10-javadoc.jar>`_
 
 Go 1.11+
 --------

documentation/sphinx/source/read-write-path.rst

@@ -0,0 +1,469 @@
+##############################
+FDB Read and Write Path
+##############################
+
+| Author: Meng Xu
+| Reviewer: Evan Tschannen, Jingyu Zhou
+| Audience: FDB developers, SREs and expert users.
+
+This document explains how FDB works at high level in database terms without mentioning FDB internal concepts.
+
+We first discuss the read path and the write path separately for a single transaction.
+We then describe how the read path and write path work together for a read and write transaction.
+In the last section, we illustrate how multiple outstanding write transactions are processed and *ordered* in FDB.
+The processing order of multiple transactions is important because it affects the parallelism of transaction processing and the write throughput.
+
+The content is based on FDB 6.2 and is true for FDB 6.3.  A new timestamp proxy role is introduced in post FDB 6.3,
+which affects the read path. We will discuss the timestamp proxy role in the future version of this document.
+
+.. image:: /images/FDB_read_path.png
+
+Components
+=================
+
+FDB is built on top of several key components.
+The terms below are common database or distributed system terms, instead of FDB specific terms.
+
+**Timestamp generator.** It serves logical time,  which defines happen-before relation:
+An event at t1 happens before another event at t2, if t1 < t2.
+The logic time is used to order events in FDB distributed systems and it is used by concurrency control to decide if two transactions have conflicts.
+The logical time is the timestamp for a transaction.
+
+* A read-only transaction has only one timestamp which is assigned when the transaction is created;
+* A read-write transaction has one timestamp at the transaction’s creation time and one timestamp at its commit time.
+
+
+**Concurrency Control.** It decides if two transactions can be executed concurrently without violating Strict Serializable Isolation (SSI) property.
+It uses the Optimistic Concurrency Control (OCC) mechanism described in [SSI] to achieve that.
+
+**Client.** It is a library, an FDB application uses, to access the database.
+It exposes the transaction concept to applications.
+Client in FDB is a *fat* client that does multiple complex operations:
+(1) It calculates read and write conflict ranges for transactions;
+(2) it batches a transaction's operations and send them all together at commit for better throughput;
+(3) it automatically retries failed transactions.
+
+**Proxies.** It is a subsystem that acts like reverse proxies to serve clients’ requests. Its main purposes is:
+
+* Serve for read request by (1) serving the logical time to client; and (2) providing which storage server has data for a key;
+* Process write transactions on behalf of clients and return the results;
+
+Each proxy has the system’s metadata, called transaction state store (txnStateStore). The metadata decides:
+(1) which key should go to which storage servers in the storage system;
+(2) which key should go to which processes in the durable queuing system;
+(3) is the database locked; etc.
+
+The metadata on all proxies are consistent at any given timestamp.
+To achieve that, when a proxy has a metadata mutation that changes the metadata at the timestamp V1,
+the mutation is propagated to all proxies (through the concurrency control component), and
+its effect is applied on all proxies before any proxy can process transactions after the timestamp V1.
+
+**Durable queuing system.** It is a queuing system for write traffic.
+Its producers are proxies that send transaction mutation data for durability purpose.
+Its consumers are storage systems that index data and serve read request.
+The queuing system is partitioned for the key-space.
+A shard (i.e., key-range) is mapped to *k* log processes in the queuing system, where *k* is the replication factor.
+The mapping between shard and storage servers decides the mapping between shard and log processes.
+
+**Storage system.** It is a collection of storage servers (SS), each of which is a sqlite database running on a single thread.
+It indexes data and serves read requests.
+Each SS has an in-memory p-tree data structure that stores the past 5-second mutations and an on-disk sqlite data.
+The in-memory data structure can serve multiple versions of key-values in the past 5 seconds.
+Due to memory limit, the in-memory data cannot hold more than 5 seconds’ multi-version key-values,
+which is the root cause why FDB’s transactions cannot be longer than 5 seconds.
+The on-disk sqlite data has only the most-recent key-value.
+
+**Zookeeper like system.** The system solves two main problems:
+
+* Store the configuration of the transaction system, which includes information such as generations of queuing systems and their processes.
+  The system used to be zookeeper. FDB later replaced it with its own implementation.
+
+* Service discovery. Processes in the zookeeper-like system serve as well-known endpoints for clients to connect to the cluster.
+  These well-known endpoint returns the list of proxies to clients.
+
+
+
+Read path of a transaction
+==================================
+
+Fig. 1 above shows a high-level view of the read path. An application uses FDB client library to read data.
+It creates a transaction and calls its read() function. The read() operation will lead to several steps.
+
+* **Step 1 (Timestamp request)**: The read operation needs a timestamp.
+  The client initiates the timestamp request through an RPC to proxy. The request will trigger Step 2 and Step 3;
+
+  * To improve throughput and reduce load on the server side, each client dynamically batches the timestamp requests.
+    A client keeps adding requests to the current batch until
+    *when* the number of requests in a batch exceeds a configurable threshold or
+    *when* the batching times out at a dynamically computed threshold.
+    Each batch sends only one timestamp request to proxy and all requests in the same batch share the same timestamp.
+
+* **Step 2 (Get latest commit version)**: When the timestamp request arrives at a proxy,
+  the proxy wants to get the largest commit version as the return value.
+  So it contacts the rest of (n-1) proxies for their latest commit versions and
+  uses the largest one as the return value for Step 1.
+
+    * O(n^2) communication cost: Because each proxy needs to contact the rest of (n-1) proxies to serve clients’ timestamp request,
+      the communication cost is n*(n-1), where n is the number of proxies;
+
+    * Batching: To reduce communication cost, each proxy batches clients’ timestamp requests for a configurable time period (say 1ms) and
+      return the same timestamp for requests in the same batch.
+
+* **Step 3 (Confirm proxy’s liveness)**: To prevent proxies that are no longer a part of the system (such as due to network partition) from serving requests,
+  each proxy contacts the queuing system for each timestamp request to confirm it is still a valid proxy
+  (i.e., not replaced by a newer generation proxy process).
+  This is based on the FDB property that at most one active queuing system is available at any given time.
+
+    * Why do we need this step? This is to achieve consensus (i.e., external consistency).
+      Compared to serializable isolation, Strict Serializable Isolation (SSI) requires external consistency.
+      It means the timestamp received by clients cannot decrease. If we do not have step and network partition happens,
+      a set of old proxies that are disconnected from the rest of systems can still serve timestamp requests to clients.
+      These timestamps can be smaller than the new generation of proxies, which breaks the external consistency in SSI.
+
+    * O(n * m) communication cost: To confirm a proxy’s liveness, the proxy has to contact all members in the queuing system to
+      ensure the queuing system is still active. This causes *m* network communication, where *m* is the number of processes in the queuing system.
+      A system with n proxies will have O(n * m) network communications at the step 3. In our deployment, n is typically equal to m;
+
+    * Do FDB production clusters have this overhead? No. Our production clusters disable the external consistency by
+      configuring the knob ALWAYS_CAUSAL_READ_RISKY.
+
+* **Step 4 (Locality request)**: The client gets which storage servers have its requested keys by sending another RPC to proxy.
+  This step returns a set of  *k* storage server interfaces, where k is the replication factor;
+
+    * Client cache mechanism: The key location will be cached in client.
+      Future requests will use the cache to directly read from storage servers,
+      which saves a trip to proxy. If location is stale, read will return error and client will retry and refresh the cache.
+
+* **Step 5 (Get data request)**: The client uses the location information from step 4 to directly query keys from corresponding storage servers.
+    * Direct read from client’s memory: If a key’s value exists in the client’s memory, the client reads it directly from its local memory.
+      This happens when a client updates a key’s value and later reads it.
+      This optimization reduces the amount of unnecessary requests to storage servers.
+
+    * Load balance: Each data exists on k storage servers, where k is the replication factor.
+      To balance the load across the k replicas, client has a load balancing algorithm to balance the number of requests to each replica.
+
+    * Transaction succeed: If the storage server has the data at the read timestamp, the client will receive the data and return succeed.
+
+    * Transaction too old error: If the read request’s timestamp is older than 5 seconds,
+      storage server may have already flushed the data from its in-memory multi-version data structure to its on-disk single-version data structure.
+      This means storage server does not have the data older than 5 seconds. So client will receive transaction too old error.
+      The client will retry with a new timestamp.
+      One scenario that can lead to the error is when it takes too long for a client to send the read request after it gets the timestamp.
+
+    * Future transaction error: Each storage server pulls data in increasing order of data’s timestamp from the queuing system.
+      Let’s define a storage server’s timestamp as the largest timestamp of data the storage server has.
+      If the read request’s timestamp is larger than the storage server’s timestamp,
+      the storage server will reply future-transaction-error to the client.
+      The client will retry. One scenario that can lead to the error is when the connection between the SS and the queuing system is slow.
+
+    * Wrong shard error: If keys in the request or result depend on data outside this storage server OR
+      if a large selector offset prevents all data from being read in one range read.
+      Client will invalidate its locality cache for the key and retry the read request at the failed key.
+
+Implementation of FDB read path
+------------------------------------------
+
+* **Step 1 (Timestamp request)**: 
+    * Each read request tries to get a timestamp if its transaction has not got one:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L2104
+    * Client batches the get-timestamp requests:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L3172
+    * Dynamic batching algorithm:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L3101-L3104
+
+* **Step 2 (Get latest commit version)**: Contacting (n-1) proxies for commit version:
+  https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1196
+
+* **Step 3 (Confirm proxy’s liveness)**:
+    * We typically set our clusters’ knob ALWAYS_CAUSAL_READ_RISKY to 1 to skip this step
+    * Proxy confirm queuing system is alive:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1199
+    * How is confirmEpochLive(..) implemented for the above item:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/TagPartitionedLogSystem.actor.cpp#L1216-L1225
+
+* **Step 4 (Locality request)**:
+  https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1312-L1313
+
+* **Step 5 (Get data request)**:
+    * Logics of handling get value request:
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1306-L1396
+    * Load balance algorithm: The loadBalance() at
+      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1342-L1344
+
+
+
+Write path of a transaction
+================================
+
+Suppose a client has a write-only transaction. Fig. 2 below shows the write path in a non-HA cluster.
+We will discuss how a transaction with both read and write works in the next section.
+
+.. image:: /images/FDB_write_path.png
+
+To simplify the explanation, the steps below do not include transaction batching on proxy,
+which is a typical database technique to increase transaction throughput.
+
+* **Step 1 (Client buffers write mutations):** Client buffers all writes in a transaction until commit is called on the transaction.
+  In the rest of document, a write is also named as a mutation.
+
+    * Client is a fat client that preprocess transactions:
+      (a) For atomic operations, if client knows the key value, it will convert atomic operations to set operations;
+      (b) For version stamp atomic operations, client adds extra bytes to key or value for the version stamp;
+      (c) If a key has multiple operations, client coalesces them to one operation whenever possible.
+
+    * How client buffers mutations:
+      https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2345-L2361
+
+* **Step 2 (Client commits the transaction):** When a client calls commit(), it performs several operations:
+
+    * **Step 2-1**: Add extra conflict ranges that are added by user but cannot be calculated from mutations.
+
+    * **Step 2-2**: Get a timestamp as the transaction’s start time. The timestamp does not need causal consistency because the transaction has no read.
+        * This request goes to one of proxies. The proxy will contact all other (n-1) proxies to get the most recent commit version as it does in read path.
+          The proxy does not need to contact log systems to confirm its activeness because it does not need causal consistency.
+
+    * **Step 2-3**:  Sends the transaction’s information to a proxy. Load balancer in client decides which proxy will be used to handle a transaction.
+      A transaction’s information includes:
+
+        * All of its mutations;
+        * Read and write conflict range;
+        * Transaction options that control a transaction’s behavior. For example, should the transaction write when the DB is locked?
+          Shall the transaction uses the first proxy in the proxy list to commit? 
+
+    * Implementation:
+        * Transaction commit function: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2895-L2899
+        * Major work of commit in client side is done at here: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2784-L2868
+        * Step 2-1: Add extra conflict ranges: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2826-L2828
+        * Step 2-2: getReadVersion at commit which does not need external consistency because we do not have read in the transaction: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2822-L2823
+        * Step 2-3: Send transaction to a proxy via RPC: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2691-L2700
+
+* When a proxy receives clients’ transactions, it commits the transaction on behalf of clients with Step 3 - 9.
+
+* **Step 3 (Proxy gets commit timestamp)**: The proxy gets the timestamp of the transaction’s commit time from the time oracle through an RPC call.
+
+    * To improve transaction throughput and reduce network communication overhead,
+      each proxy dynamically batch transactions and process transactions in batches.
+      A proxy keeps batching transactions until the batch time exceeds a configurable timeout value or
+      until the number of transactions exceed a configurable value or
+      until the total bytes of the batch exceeds a dynamically calculated desired size.
+
+    * The network overhead is 1 network communication per batch of commit transactions;
+
+    * How is the dynamically calculated batch size calculated: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1770-L1774
+    * How commit transactions are batched: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L416-L486
+    * How each transaction batch is handled: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L523-L1174
+    * Where does proxy sends commit timestamp request to the timestamp generator:  https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L586-L587
+
+* **Step 4 (Proxy builds transactions’ conflict ranges)**: Because the concurrency control component may have multiple processes,
+  each of which is responsible for resolving conflicts in a key range,
+  the proxy needs to build one transaction-conflict-resolution request for each concurrency control process:
+  For each transaction, the proxy splits its read and write conflict ranges based on concurrency control process’ responsible ranges.
+  The proxy will create k conflict resolution requests for each transaction, where k is the number of processes in the concurrency control component.
+
+    * Implementation: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbserver/MasterProxyServer.actor.cpp#L607-L618
+
+* **Step 5 (Proxy sends conflict resolution requests to concurrency control)**:
+  Each concurrency control process is responsible for checking conflicts in a key range.
+  Each process checks if the transaction has conflicts with other transactions in its  key-range.
+  Each process returns the conflict checking result back to the proxy.
+
+    * What is conflict range?
+        * A transaction’s write conflict range includes any key and key-ranges that are modified in the transactions.
+        * A transaction’s read conflict range includes any key and key-ranges that are read in the transaction.
+        * Client can also use transaction options to add explicit read-conflict-range or write-conflict-range.
+          Example: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L2634-L2635
+
+    * **Piggy-back metadata change**. If the transaction changes database’s metadata, such as locking the database,
+      the change is considered as a special mutation and also checked for conflicts by the concurrency control component.
+      The primary difference between metadata mutation and normal mutations is that the metadata change must be propagated to all proxies
+      so that all proxies have a consistent view of database’s metadata.
+      This is achieved by piggy-backing metadata change in the reply from resolver to proxies.
+
+    * Implementation
+        * Create conflict resolution requests for a batch of transactions: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L607-L618
+        * Metadata mutations are sent from proxy to concurrency control processes: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L366-L369
+
+* **Step 6 (Resolve conflicts among concurrent transactions)**:
+  Each concurrency control process checks conflicts among transactions based on the theory in [1].
+  In a nutshell, it checks for read-write conflicts. Suppose two transactions operates on the same key.
+  If a write transaction’s time overlaps between another read-write transaction’s start time and commit time,
+  only one transaction can commit: the one that arrives first at all concurrency control processes will commit.
+
+    * Implementation
+        * Proxy sends conflict checking request: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L626-L629
+        * Concurrency control process handles the request: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/Resolver.actor.cpp#L320-L322
+
+* **Step 7 (Proxy’s post resolution processing)**:
+  Once the proxy receives conflict-resolution replies from all concurrency control processes, it performs three steps
+
+    * **Step 7-1 (Apply metadata effect caused by other proxies)**: As mentioned above, when a proxy changes database’s metadata,
+      the metadata mutations will be propagated via the concurrency control component to other proxies.
+      So the proxy needs to first compute and apply these metadata mutations onto the proxy’s local states.
+      Otherwise, the proxy will operate in a different view of database’s metadata.
+
+        * For example, if one proxy locks the database in a committed transaction at time t1, all other proxies should have seen the lock immediately after t1. Since another proxy may have transactions in flight already at t1, the proxy must first apply the “lock“ effect before it can process its in-flight transactions.
+        * How metadata effect is applied in implementation:  https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L678-L719
+
+    * **Step 7-2 (Determine which transactions are committed)**: Proxy combines results from all concurrency control processes.
+      Only if all concurrency control processes say a transaction is committed, will the transaction be considered as committed by the proxy.
+
+        * Implementation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L721-L757
+
+    * **Step 7-3 (Apply metadata effect caused by this proxy)**: For each committed transaction,
+      this proxy applies its metadata mutations to the proxy’s local state.
+
+        * Note: These metadata mutations are also sent to concurrency control processes and propagated to other proxies at Step 5.
+          This step is to apply metadata effect on its own proxy’s states.
+        * Implementation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L763-L777
+
+    * **Step 7-4 (Assign mutations to storage servers and serialize them)**:
+      In order to let the rest of system (the queuing system and storage system) know which process a mutation should be routed to,
+      the proxy needs to add tags to mutations.
+      The proxy serializes mutations with the same tag into the same message and sends the serialized message to the queuing system.
+
+        * Implementation of adding tags and serializing mutations into messages: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L800-L910
+        * The lines that add tags to a mutation and serialize it: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L846-L847
+
+    * **Step 7-5 (Duplicate and serialize mutations to backup system keyspace)**:
+      When backup or disaster recovery (DR) is enabled, each proxy captures mutation streams into a dedicated system keyspace.
+      Mutations in a transaction batch are serialized as a single mutation in a dedicated system keyspace.
+
+        * How mutations are duplicated for backup and DR: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L912-L986
+        * Note: FDB will have a new backup system that avoids duplicating mutations to the system keyspace.
+          Its design is similar to database’s Change Data Capture (CDC) design. The new backup system is not production-ready yet.
+
+* **Step 8 (Make mutation messages durable in the queuing system)**:
+  Proxy sends serialized mutation messages to the queuing system.
+  The queuing system will append the mutation to an append-only file, fsync it, and send the respnose back.
+  Each message has a tag, which decides which process in the queuing system the message should be sent to.
+  The queuing system returns to the proxy the minimum known committed version, which is the smallest commit version among all proxies.
+  The minimum known commit version is used when the system recovers from fault.
+
+    * Sending messages to the queuing system is abstracted into a push() operation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1045
+    * The minimum known committed version is called minKnownCommittedVersion. It is updated for each commit: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1067
+
+* **Step 9 (Reply to client)**: Proxy replies the transaction’s result to client.
+  If the transaction fails (say due to transaction conflicts), proxy sends the error message to the client.
+
+    * Reply to clients based on different transaction’s results: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1117-L1138
+
+* **Step 10 (Storage systems pull data from queuing system)**:
+  Storage system asynchronously pulls data from queuing system and indexes data for read path.
+
+    * Each SS has a primary process (called primary tLog) in the queuing system to pull data from the SS’s data from the queuing system.
+      Each SS only gets in-ordered streams of mutations that are owned by the SS.
+
+    * In failure scenario when a SS cannot reach the primary tLog, the SS will pull data from different tLogs that have part of the SS’s data.
+      The SS will then merge the stream of data from different tLogs.
+
+    * Each SS does not make its pulled data durable to disk until the data becomes
+      at least 5 seconds older than the most recent data the SS has pulled.
+      This allows each SS to roll back at least 5 seconds of mutations.
+
+    * Why do we need roll back feature for SS? This comes from an optimization used in FDB.
+      To make a mutation available in a SS as soon as possible,
+      a SS may fetch a mutation from the queuing system that has not been fully replicated.
+      The mutation’s transaction may be aborted in rare situations, such as
+      when FDB has to recover from faults and decides to throw away the last few non-fully-durable transactions.
+      SSes must throw away data in the aborted transactions.
+
+    * Why does SS not make data durable until 5 seconds later?
+      This is because today’s SS does not support rolling back data that has already been made durable on disk.
+      To support roll back, SS keeps data that might be rolled back in memory.
+      When roll-back is needed, SS just throws away the in-memory data. This simplifies the SS implementation.
+
+
+    * Each storage process pulls data from the queuing system: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/storageserver.actor.cpp#L3593-L3599
+
+
+
+Read write path of a transaction
+====================================
+
+This section uses an example transaction to describe how a transaction with both read and write operation works in FDB.
+
+Suppose application creates the following transaction, where *Future<int>* is an object that holds an asynchronous call and
+becomes ready when the async call returns, and *wait()* is a synchronous point when the code waits for futures to be ready.
+The following code reads key k1 and k2 from database,  increases k1’s value by 1 and write back k1’s new value into database.
+
+**Example Transaction** ::
+
+    Line1: Transaction tr;
+    Line2: Future<int> fv1 = tr.get(k1);
+    Line3: Future<int> fv2 = tr.get(k2);
+    Line4: v1 = wait(fv1);
+    Line5: v2 = wait(fv2);
+    Line6: tr.set(v1+v2);
+    Line7: tr.commit();
+
+The transaction starts with the read path:
+
+* When tr.get() is called, FDB client issues a timestamp request to proxies *if* the transaction has not set its start timestamp.
+  The logic is the Step 1 in the read path;
+
+* Batching timestamp requests. When another tr.get() is called, it will try to get a timestamp as well. If we let every get request to follow the Step 1 in the read path, the performance overhead (especially network communication) will be a lot. In addition, this is not necessary because a transaction has only one start timestamp. To solve this problem, client chooses to batch timestamp requests from the same transaction and only issues one timestamp request when the transaction size reaches a preconfigured threshold or when the transaction duration reaches the batching timeout threshold. 
+    * Timestamp requests are batched: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbclient/NativeAPI.actor.cpp#L3185
+    * Thresholds for client to send the timestamp request: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbclient/NativeAPI.actor.cpp#L3095-L3098
+
+* Each read request, i.e., tr.get operation in the example, will follow the read path to get data from storage servers, except that they will share the same timestamp;
+    * These read requests are sent to FDB cluster in parallel.
+      The ordering of which read request will be ready first depends on requests’ network path and storage servers’ load.
+    * In the example, tr.get(k2) may return result earlier than tr.get(k1).
+
+* Client will likely block at the synchronization point at Line 4, until the value is returned from the cluster.
+    * To maximize clients’ performance, a client can issue multiple transactions concurrently.
+      When one transaction is blocked at the synchronization point,
+      the client can switch to work on the other transactions concurrently.
+
+* Client may or may not block at the synchronization point at Line 5.
+  If tr.get(k2) returns earlier than tr.get(k1), the future fv2 is already ready when the client arrives at Line 5.
+
+* At Line 6, client starts the write path. Because the transaction already has its start timestamp,
+  client does not need to request for the transaction’s start time any more and can skip the Step 2-2 in the write path.
+
+* At Line 7, client commits the transaction, which will trigger the operations from Step 2 in the write path.
+
+
+A transaction can get more complex than the example above.
+
+* A transaction can have more writes operations between Line 6 and Line 7.
+  Those writes will be buffered in client’s memory, which is the Step 1 in the write path.
+  Only when the client calls commit(), will the rest of steps in the write path will be triggered;
+
+* A transaction can have reads operations between Line 6 and Line 7 as well.
+
+* A transaction may return commit_unknown_result, which indicate the transaction may or may not succeed.
+  If application simply retries the transaction, the transaction may get executed twice.
+  To solve this problem, the application can adds a transaction id to the transaction and
+  check if the transaction id exists on the commit_unknown_result error.
+
+
+
+Concurrency and ordering of multiple write transactions
+=======================================================================
+
+FDB orders concurrent transactions in increasing order of the transactions’ commit timestamp.
+The ordering is enforced in the timestamp generator, the concurrency control component and the durable queuing system.
+
+* When timestamp generator serves the commit timestamp request from a proxy,
+  the reply includes not only the commit timestamp but also the latest commit timestamp the generator has sent out.
+  For example, the timestamp generator just gave out the commit timestamp 50.
+  When the next request arrives, the generator’s timestamp is 100 and the generator replies (50, 100).
+  When the second request arrives and the generator’s timestamp is 200, the generator replies (100, 200).
+
+* When a proxy sends conflict resolution requests to concurrency control processes or durable requests to the queuing system,
+  each request includes both the current transaction’s commit timestamp and the previous transaction’s commit timestamp.
+
+* Each concurrency control process and each process in the queuing system always process requests in the strict order of the request’s commit version.
+  The semantics is do not process a request whose commit timestamp is V2 until the request at its previous commit timestamp V1 has been processed.
+
+
+We use the following example and draw its swimlane diagram to illustrate how two write transactions are ordered in FDB.
+The diagram with notes can be viewed at `here <https://lucid.app/lucidchart/6336dbe3-cff4-4c46-995a-4ca3d9260696/view?page=0_0#?folder_id=home&browser=icon>`_.
+
+.. image:: /images/FDB_multiple_txn_swimlane_diagram.png
+
+Reference
+============
+
+[SSI] Serializable Snapshot Isolation in PostgreSQL. https://arxiv.org/pdf/1208.4179.pdf

documentation/sphinx/source/release-notes/release-notes-620.rst

@@ -6,6 +6,11 @@ Release Notes
 ======
 * Fix invalid memory access on data distributor when snapshotting large clusters. `(PR #4076) <https://github.com/apple/foundationdb/pull/4076>`_
 * Add human-readable DateTime to trace events `(PR #4087) <https://github.com/apple/foundationdb/pull/4087>`_
+* Proxy rejects transaction batch that exceeds MVCC window `(PR #4113) <https://github.com/apple/foundationdb/pull/4113>`_
+* Add a command in fdbcli to manually trigger the detailed teams information loggings in data distribution. `(PR #4060) <https://github.com/apple/foundationdb/pull/4060>`_
+* Add documentation on read and write Path. `(PR #4099) <https://github.com/apple/foundationdb/pull/4099>`_
+* Add a histogram to expose commit batching window on Proxies. `(PR #4166) <https://github.com/apple/foundationdb/pull/4166>`_
+* Fix double counting of range reads in TransactionMetrics. `(PR #4130) <https://github.com/apple/foundationdb/pull/4130>`_
 
 6.2.28
 ======

documentation/sphinx/source/release-notes/release-notes-630.rst

@@ -4,6 +4,12 @@ Release Notes
 
 6.3.10
 ======
+* Make fault tolerance metric calculation in HA clusters consistent with 6.2 branch. `(PR #4175) <https://github.com/apple/foundationdb/pull/4175>`_
+* Bug fix, stack overflow in redwood storage engine. `(PR #4161) <https://github.com/apple/foundationdb/pull/4161>`_
+* Bug fix, getting certain special keys fail. `(PR #4128) <https://github.com/apple/foundationdb/pull/4128>`_ 
+* Prevent slow task on TLog by yielding while processing ignored pop requests. `(PR #4112) <https://github.com/apple/foundationdb/pull/4112>`_
+* Support reading xxhash3 sqlite checksums. `(PR #4104) <https://github.com/apple/foundationdb/pull/4104>`_
+* Fix a race between submit and abort backup. `(PR #3935) <https://github.com/apple/foundationdb/pull/3935>`_
 
 Packaging
 ---------
@@ -132,7 +138,7 @@ Fixes from previous versions
 * The 6.3.3 patch release includes all fixes from the patch release 6.2.23. :doc:`(6.2 Release Notes) </release-notes/release-notes-620>`
 * The 6.3.5 patch release includes all fixes from the patch releases 6.2.24 and 6.2.25. :doc:`(6.2 Release Notes) </release-notes/release-notes-620>`
 * The 6.3.9 patch release includes all fixes from the patch releases 6.2.26. :doc:`(6.2 Release Notes) </release-notes/release-notes-620>`
-* The 6.3.10 patch release includes all fixes from the patch releases 6.2.27. :doc:`(6.2 Release Notes) </release-notes/release-notes-620>`
+* The 6.3.10 patch release includes all fixes from the patch releases 6.2.27-6.2.29 :doc:`(6.2 Release Notes) </release-notes/release-notes-620>`
 
 Fixes only impacting 6.3.0+
 ---------------------------

documentation/sphinx/source/release-notes/release-notes-700.rst

@@ -35,7 +35,8 @@ Status
 Bindings
 --------
 * Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
-
+* C: Added a function, ``fdb_database_reboot_worker``, to reboot or suspend the specified process. `(PR #4094) <https://github.com/apple/foundationdb/pull/4094>`_
+* C: Added a function, ``fdb_database_force_recovery_with_data_loss``, to force the database to recover into the given datacenter. `(PR #4420) <https://github.com/apple/foundationdb/pull/4220>`_
 
 Other Changes
 -------------

documentation/sphinx/source/technical-overview.rst

@@ -28,6 +28,8 @@ These documents explain the engineering design of FoundationDB, with detailed in
 
 * :doc:`kv-architecture` provides a description of every major role a process in FoundationDB can fulfill.
 
+* :doc:`read-write-path` describes how FDB read and write path works.
+
 .. toctree::
    :maxdepth: 1
    :titlesonly:
@@ -45,3 +47,4 @@ These documents explain the engineering design of FoundationDB, with detailed in
    flow
    testing
    kv-architecture
+   read-write-path

fdbbackup/FileConverter.actor.cpp

@@ -31,6 +31,7 @@
 #include "fdbclient/MutationList.h"
 #include "flow/flow.h"
 #include "flow/serialize.h"
+#include "fdbclient/BuildFlags.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 namespace file_converter {
@@ -51,12 +52,17 @@ void printConvertUsage() {
 	          << "  --trace_format FORMAT\n"
 	          << "                  Select the format of the trace files. xml (the default) and json are supported.\n"
 	          << "                  Has no effect unless --log is specified.\n"
+	          << "  --build_flags   Print build information and exit.\n"
 	          << "  -h, --help      Display this help and exit.\n"
 	          << "\n";
 
 	return;
 }
 
+void printBuildInformation() {
+	printf("%s", jsonBuildInformation().c_str());
+}
+
 void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
 	std::cout << msg << " " << files.size() << " log files\n";
 	for (const auto& file : files) {
@@ -535,6 +541,10 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
 		case OPT_TRACE_LOG_GROUP:
 			param->trace_log_group = args->OptionArg();
 			break;
+		case OPT_BUILD_FLAGS:
+			printBuildInformation();
+			return FDB_EXIT_ERROR;
+			break;
 		}
 	}
 	return FDB_EXIT_SUCCESS;

fdbbackup/FileConverter.h

@@ -38,6 +38,7 @@ enum {
 	OPT_TRACE_FORMAT,
 	OPT_TRACE_LOG_GROUP,
 	OPT_INPUT_FILE,
+	OPT_BUILD_FLAGS,
 	OPT_HELP
 };
 
@@ -54,6 +55,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
 	                                        { OPT_INPUT_FILE, "-i", SO_REQ_SEP },
 	                                        { OPT_INPUT_FILE, "--input", SO_REQ_SEP },
+	                                        { OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
 	                                        { OPT_HELP, "-?", SO_NONE },
 	                                        { OPT_HELP, "-h", SO_NONE },
 	                                        { OPT_HELP, "--help", SO_NONE },

fdbbackup/FileDecoder.actor.cpp

@@ -28,6 +28,7 @@
 #include "fdbclient/MutationList.h"
 #include "flow/flow.h"
 #include "flow/serialize.h"
+#include "fdbclient/BuildFlags.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 #define SevDecodeInfo SevVerbose
@@ -41,10 +42,15 @@ void printDecodeUsage() {
 	             "  -r, --container   Container URL.\n"
 	             "  -i, --input FILE  Log file to be decoded.\n"
 	             "  --crash           Crash on serious error.\n"
+	             "  --build_flags     Print build information and exit.\n"
 	             "\n";
 	return;
 }
 
+void printBuildInformation() {
+	printf("%s", jsonBuildInformation().c_str());
+}
+
 struct DecodeParams {
 	std::string container_url;
 	std::string file;
@@ -121,6 +127,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 		case OPT_TRACE_LOG_GROUP:
 			param->trace_log_group = args->OptionArg();
 			break;
+		case OPT_BUILD_FLAGS:
+			printBuildInformation();
+			return FDB_EXIT_ERROR;
+			break;
 		}
 	}
 	return FDB_EXIT_SUCCESS;
@@ -201,6 +211,15 @@ struct VersionedMutations {
 	Arena arena; // The arena that contains the mutations.
 };
 
+struct VersionedKVPart {
+	Arena arena;
+	Version version;
+	int32_t part;
+	StringRef kv;
+	VersionedKVPart(Arena arena, Version version, int32_t part, StringRef kv)
+	  : arena(arena), version(version), part(part), kv(kv) {}
+};
+
 /*
  * Model a decoding progress for a mutation file. Usage is:
  *
@@ -217,7 +236,10 @@ struct VersionedMutations {
  * pairs, the decoding of mutation batch needs to look ahead one more pair. So
  * at any time this object might have two blocks of data in memory.
  */
-struct DecodeProgress {
+class DecodeProgress {
+	std::vector<VersionedKVPart> keyValues;
+
+public:
 	DecodeProgress() = default;
 	template <class U>
 	DecodeProgress(const LogFile& file, U &&values)
@@ -227,9 +249,9 @@ struct DecodeProgress {
 	// However, we could have unfinished version in the buffer when EOF is true,
 	// which means we should look for data in the next file. The caller
 	// should call getUnfinishedBuffer() to get these left data.
-	bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); }
+	bool finished() const { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); }
 
-	std::vector<std::tuple<Arena, Version, int32_t, StringRef>>&& getUnfinishedBuffer() && { return std::move(keyValues); }
+	std::vector<VersionedKVPart>&& getUnfinishedBuffer() && { return std::move(keyValues); }
 
 	// Returns all mutations of the next version in a batch.
 	Future<VersionedMutations> getNextBatch() { return getNextBatchImpl(this); }
@@ -239,7 +261,7 @@ struct DecodeProgress {
 	// The following are private APIs:
 
 	// Returns true if value contains complete data.
-	bool isValueComplete(StringRef value) {
+	static bool isValueComplete(StringRef value) {
 		StringRefReader reader(value, restore_corrupted_data());
 
 		reader.consume<uint64_t>(); // Consume the includeVersion
@@ -260,41 +282,41 @@ struct DecodeProgress {
 				wait(readAndDecodeFile(self));
 			}
 
-			auto& tuple = self->keyValues[0];
+			const auto& kv = self->keyValues[0];
-			ASSERT(std::get<2>(tuple) == 0); // first part number must be 0.
+			ASSERT(kv.part == 0);
 
 			// decode next versions, check if they are continuous parts
 			int idx = 1; // next kv pair in "keyValues"
-			int bufSize = std::get<3>(tuple).size();
+			int bufSize = kv.kv.size();
 			for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
 				if (idx == self->keyValues.size()) break;
 
-				auto next_tuple = self->keyValues[idx];
+				const auto& nextKV = self->keyValues[idx];
-				if (std::get<1>(tuple) != std::get<1>(next_tuple)) {
+				if (kv.version != nextKV.version) {
 					break;
 				}
 
-				if (lastPart + 1 != std::get<2>(next_tuple)) {
+				if (lastPart + 1 != nextKV.part) {
-					TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", std::get<2>(next_tuple));
+					TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", nextKV.part);
 					throw restore_corrupted_data();
 				}
-				bufSize += std::get<3>(next_tuple).size();
+				bufSize += nextKV.kv.size();
 			}
 
 			VersionedMutations m;
-			m.version = std::get<1>(tuple);
+			m.version = kv.version;
 			TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size());
-			StringRef value = std::get<3>(tuple);
+			StringRef value = kv.kv;
 			if (idx > 1) {
 				// Stitch parts into one and then decode one by one
 				Standalone<StringRef> buf = self->combineValues(idx, bufSize);
 				value = buf;
 				m.arena = buf.arena();
 			}
-			if (self->isValueComplete(value)) {
+			if (isValueComplete(value)) {
 				m.mutations = decode_value(value);
 				if (m.arena.getSize() == 0) {
-					m.arena = std::get<0>(tuple);
+					m.arena = kv.arena;
 				}
 				self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
 				return m;
@@ -317,7 +339,7 @@ struct DecodeProgress {
 		Standalone<StringRef> buf = makeString(len);
 		int n = 0;
 		for (int i = 0; i < idx; i++) {
-			const auto& value = std::get<3>(keyValues[i]);
+			const auto& value = keyValues[i].kv;
 			memcpy(mutateString(buf) + n, value.begin(), value.size());
 			n += value.size();
 		}
@@ -363,12 +385,9 @@ struct DecodeProgress {
 			// The (version, part) in a block can be out of order, i.e., (3, 0)
 			// can be followed by (4, 0), and then (3, 1). So we need to sort them
 			// first by version, and then by part number.
-			std::sort(keyValues.begin(), keyValues.end(),
+			std::sort(keyValues.begin(), keyValues.end(), [](const VersionedKVPart& a, const VersionedKVPart& b) {
-			          [](const std::tuple<Arena, Version, int32_t, StringRef>& a,
+				return a.version == b.version ? a.part < b.part : a.version < b.version;
-			             const std::tuple<Arena, Version, int32_t, StringRef>& b) {
+			});
-				          return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b)
-				                                                  : std::get<1>(a) < std::get<1>(b);
-			          });
 			return;
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin());
@@ -419,8 +438,6 @@ struct DecodeProgress {
 	int64_t offset = 0;
 	bool eof = false;
 	bool leftover = false; // Done but has unfinished version batch data left
-	// A (version, part_number)'s mutations and memory arena.
-	std::vector<std::tuple<Arena, Version, int32_t, StringRef>> keyValues;
 };
 
 ACTOR Future<Void> decode_logs(DecodeParams params) {
@@ -445,7 +462,7 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
 
 	state int i = 0;
 	// Previous file's unfinished version data
-	state std::vector<std::tuple<Arena, Version, int32_t, StringRef>> left;
+	state std::vector<VersionedKVPart> left;
 	for (; i < logs.size(); i++) {
 		if (logs[i].fileSize == 0) continue;
 

fdbcli/CMakeLists.txt

@@ -11,9 +11,11 @@ endif()
 add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
 target_link_libraries(fdbcli PRIVATE fdbclient)
 
-if(GENERATE_DEBUG_PACKAGES)
+if(NOT OPEN_FOR_IDE)
-  fdb_install(TARGETS fdbcli DESTINATION bin COMPONENT clients)
+  if(GENERATE_DEBUG_PACKAGES)
-else()
+    fdb_install(TARGETS fdbcli DESTINATION bin COMPONENT clients)
-  add_custom_target(prepare_fdbcli_install ALL DEPENDS strip_only_fdbcli)
+  else()
-  fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbcli DESTINATION bin COMPONENT clients)
+    add_custom_target(prepare_fdbcli_install ALL DEPENDS strip_only_fdbcli)
+    fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbcli DESTINATION bin COMPONENT clients)
+  endif()
 endif()

fdbcli/fdbcli.actor.cpp

@@ -51,6 +51,7 @@
 #endif
 
 #include "fdbclient/versions.h"
+#include "fdbclient/BuildFlags.h"
 
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
@@ -70,6 +71,7 @@ enum {
 	OPT_NO_HINTS,
 	OPT_STATUS_FROM_JSON,
 	OPT_VERSION,
+	OPT_BUILD_FLAGS,
 	OPT_TRACE_FORMAT,
 	OPT_KNOB,
 	OPT_DEBUG_TLS
@@ -90,6 +92,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
 	                                  { OPT_STATUS_FROM_JSON, "--status-from-json", SO_REQ_SEP },
 	                                  { OPT_VERSION, "--version", SO_NONE },
 	                                  { OPT_VERSION, "-v", SO_NONE },
+	                                  { OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
 	                                  { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
 	                                  { OPT_KNOB, "--knob_", SO_REQ_SEP },
 	                                  { OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
@@ -152,17 +155,16 @@ public:
 
 	//Applies all enabled transaction options to the given transaction
 	void apply(Reference<ReadYourWritesTransaction> tr) {
-		for(auto itr = transactionOptions.options.begin(); itr != transactionOptions.options.end(); ++itr)
+		for (const auto& [name, value] : transactionOptions.options) {
-			tr->setOption(itr->first, itr->second.castTo<StringRef>());
+			tr->setOption(name, value.castTo<StringRef>());
+		}
 	}
 
 	//Returns true if any options have been set
-	bool hasAnyOptionsEnabled() {
+	bool hasAnyOptionsEnabled() const { return !transactionOptions.options.empty(); }
-		return !transactionOptions.options.empty();
-	}
 
 	//Prints a list of enabled options, along with their parameters (if any)
-	void print() {
+	void print() const {
 		bool found = false;
 		found = found || transactionOptions.print();
 
@@ -171,14 +173,10 @@ public:
 	}
 
 	//Returns a vector of the names of all documented options
-	std::vector<std::string> getValidOptions() {
+	std::vector<std::string> getValidOptions() const { return transactionOptions.getValidOptions(); }
-		return transactionOptions.getValidOptions();
-	}
 
 	//Prints the help string obtained by invoking `help options'
-	void printHelpString() {
+	void printHelpString() const { transactionOptions.printHelpString(); }
-		transactionOptions.printHelpString();
-	}
 
 private:
 	//Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction.
@@ -219,7 +217,7 @@ private:
 		}
 
 		//Prints a list of all enabled options in this group
-		bool print() {
+		bool print() const {
 			bool found = false;
 
 			for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) {
@@ -238,7 +236,7 @@ private:
 		}
 
 		//Returns true if the specified option is documented
-		bool isDocumented(typename T::Option option) {
+		bool isDocumented(typename T::Option option) const {
 			FDBOptionInfo info = T::optionInfo.getMustExist(option);
 
 			std::string deprecatedStr = "Deprecated";
@@ -246,7 +244,7 @@ private:
 		}
 
 		//Returns a vector of the names of all documented options
-		std::vector<std::string> getValidOptions() {
+		std::vector<std::string> getValidOptions() const {
 			std::vector<std::string> ret;
 
 			for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr)
@@ -258,7 +256,7 @@ private:
 
 		//Prints a help string for each option in this group. Any options with no comment
 		//are excluded from this help string. Lines are wrapped to 80 characters.
-		void printHelpString() {
+		void printHelpString() const {
 			for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) {
 				if(isDocumented(itr->second)) {
 					FDBOptionInfo info = T::optionInfo.getMustExist(itr->second);
@@ -433,6 +431,7 @@ static void printProgramUsage(const char* name) {
 	       "                 Changes a knob option. KNOBNAME should be lowercase.\n"
 				 "  --debug-tls    Prints the TLS configuration and certificate chain, then exits.\n"
 				 "                 Useful in reporting and diagnosing TLS issues.\n"
+	       "  --build_flags  Print build information and exit.\n"
 	       "  -v, --version  Print FoundationDB CLI version information and exit.\n"
 	       "  -h, --help     Display this help and exit.\n");
 }
@@ -615,6 +614,9 @@ void initHelp() {
 	    CommandHelp("unlock <UID>", "unlock the database with the provided lockUID",
 	                "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the "
 	                "user will be asked to enter a passphrase to confirm their intent.");
+	helpMap["triggerddteaminfolog"] =
+	    CommandHelp("triggerddteaminfolog", "trigger the data distributor teams logging",
+	                "Trigger the data distributor to log detailed information about its teams.");
 
 	hiddenCommands.insert("expensive_data_check");
 	hiddenCommands.insert("datadistribution");
@@ -627,11 +629,15 @@ void printVersion() {
 	printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
 }
 
+void printBuildInformation() {
+	printf("%s", jsonBuildInformation().c_str());
+}
+
 void printHelpOverview() {
 	printf("\nList of commands:\n\n");
-	for (auto i = helpMap.begin(); i != helpMap.end(); ++i)
+	for (const auto& [command, help] : helpMap) {
-		if (i->second.short_desc.size())
+		if (help.short_desc.size()) printf(" %s:\n      %s\n", command.c_str(), help.short_desc.c_str());
-			printf(" %s:\n      %s\n", i->first.c_str(), i->second.short_desc.c_str());
+	}
 	printf("\nFor information on a specific command, type `help <command>'.");
 	printf("\nFor information on escaping keys and values, type `help escaping'.");
 	printf("\nFor information on available options, type `help options'.\n\n");
@@ -1774,6 +1780,23 @@ int printStatusFromJSON( std::string const& jsonFileName ) {
 	}
 }
 
+ACTOR Future<Void> triggerDDTeamInfoLog(Database db) {
+	state ReadYourWritesTransaction tr(db);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			std::string v = deterministicRandom()->randomUniqueID().toString();
+			tr.set(triggerDDTeamInfoPrintKey, v);
+			wait(tr.commit());
+			printf("Triggered team info logging in data distribution.\n");
+			return Void();
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
 ACTOR Future<Void> timeWarning( double when, const char* msg ) {
 	wait( delay(when) );
 	fputs( msg, stderr );
@@ -2005,16 +2028,18 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDa
 		configString = "new";
 	}
 
-	for(auto kv : configJSON) {
+	for (const auto& [name, value] : configJSON) {
 		if(!configString.empty()) {
 			configString += " ";
 		}
-		if( kv.second.type() == json_spirit::int_type ) {
+		if (value.type() == json_spirit::int_type) {
-			configString += kv.first + ":=" + format("%d", kv.second.get_int());
+			configString += name + ":=" + format("%d", value.get_int());
-		} else if( kv.second.type() == json_spirit::str_type ) {
+		} else if (value.type() == json_spirit::str_type) {
-			configString += kv.second.get_str();
+			configString += value.get_str();
-		} else if( kv.second.type() == json_spirit::array_type ) {
+		} else if (value.type() == json_spirit::array_type) {
-			configString += kv.first + "=" + json_spirit::write_string(json_spirit::mValue(kv.second.get_array()), json_spirit::Output_options::none);
+			configString +=
+			    name + "=" +
+			    json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none);
 		} else {
 			printUsage(LiteralStringRef("fileconfigure"));
 			return true;
@@ -2229,8 +2254,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 		}
 
 		printf("There are currently %zu servers or processes being excluded from the database:\n", excl.size());
-		for(auto& e : excl)
+		for (const auto& e : excl) printf("  %s\n", e.toString().c_str());
-			printf("  %s\n", e.toString().c_str());
 
 		printf("To find out whether it is safe to remove one or more of these\n"
 			   "servers from the cluster, type `exclude <addresses>'.\n"
@@ -2435,7 +2459,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 
 		bool foundCoordinator = false;
 		auto ccs = ClusterConnectionFile( ccf->getFilename() ).getConnectionString();
-		for( auto& c : ccs.coordinators()) {
+		for (const auto& c : ccs.coordinators()) {
 			if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
 			    std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
 				printf("WARNING: %s is a coordinator!\n", c.toString().c_str());
@@ -2483,7 +2507,7 @@ ACTOR Future<bool> setClass( Database db, std::vector<StringRef> tokens ) {
 		std::sort(workers.begin(), workers.end(), ProcessData::sort_by_address());
 
 		printf("There are currently %zu processes in the database:\n", workers.size());
-		for(auto& w : workers)
+		for (const auto& w : workers)
 			printf("  %s: %s (%s)\n", w.address.toString().c_str(), w.processClass.toString().c_str(), w.processClass.sourceString().c_str());
 		return false;
 	}
@@ -2833,38 +2857,33 @@ struct CLIOptions {
 			return;
 		}
 
-		delete FLOW_KNOBS;
+		for (const auto& [knob, value] : knobs) {
-		FlowKnobs* flowKnobs = new FlowKnobs;
-		FLOW_KNOBS = flowKnobs;
-
-		delete CLIENT_KNOBS;
-		ClientKnobs* clientKnobs = new ClientKnobs;
-		CLIENT_KNOBS = clientKnobs;
-
-		for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
 			try {
-				if (!flowKnobs->setKnob( k->first, k->second ) &&
+				if (!globalFlowKnobs->setKnob(knob, value) && !globalClientKnobs->setKnob(knob, value)) {
-					!clientKnobs->setKnob( k->first, k->second ))
+					fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", knob.c_str());
-				{
+					TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knob));
-					fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
-					TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
 				}
 			} catch (Error& e) {
 				if (e.code() == error_code_invalid_option_value) {
-					fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", k->second.c_str(), k->first.c_str());
+					fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", value.c_str(), knob.c_str());
-					TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second));
+					TraceEvent(SevWarnAlways, "InvalidKnobValue")
+					    .detail("Knob", printable(knob))
+					    .detail("Value", printable(value));
 				}
 				else {
-					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what());
+					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knob.c_str(), e.what());
-					TraceEvent(SevError, "FailedToSetKnob").detail("Knob", printable(k->first)).detail("Value", printable(k->second)).error(e);
+					TraceEvent(SevError, "FailedToSetKnob")
+					    .detail("Knob", printable(knob))
+					    .detail("Value", printable(value))
+					    .error(e);
 					exit_code = FDB_EXIT_ERROR;
 				}
 			}
 		}
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		flowKnobs->initialize(true);
+		globalFlowKnobs->initialize(true);
-		clientKnobs->initialize(true);
+		globalClientKnobs->initialize(true);
 	}
 
 	int processArg(CSimpleOpt& args) {
@@ -2949,6 +2968,9 @@ struct CLIOptions {
 			case OPT_VERSION:
 				printVersion();
 				return FDB_EXIT_SUCCESS;
+			case OPT_BUILD_FLAGS:
+				printBuildInformation();
+				return FDB_EXIT_SUCCESS;
 		}
 		return -1;
 	}
@@ -3240,6 +3262,11 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "triggerddteaminfolog")) {
+					wait(triggerDDTeamInfoLog(db));
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "configure")) {
 					bool err = wait(configure(db, tokens, db->getConnectionFile(), &linenoise, warn));
 					if (err) is_error = true;

fdbclient/AsyncFileS3BlobStore.actor.h

@@ -54,10 +54,10 @@ static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
 // using multi-part upload and beginning to transfer each part as soon as it is large enough.
 // All write operations file operations must be sequential and contiguous.
 // Limits on part sizes, upload speed, and concurrent uploads are taken from the S3BlobStoreEndpoint being used.
-class AsyncFileS3BlobStoreWrite : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreWrite> {
+class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreWrite> {
 public:
-	virtual void addref() { ReferenceCounted<AsyncFileS3BlobStoreWrite>::addref(); }
+	void addref() override { ReferenceCounted<AsyncFileS3BlobStoreWrite>::addref(); }
-	virtual void delref() { ReferenceCounted<AsyncFileS3BlobStoreWrite>::delref(); }
+	void delref() override { ReferenceCounted<AsyncFileS3BlobStoreWrite>::delref(); }
 
 	struct Part : ReferenceCounted<Part> {
 		Part(int n, int minSize)
@@ -256,10 +256,10 @@ public:
 };
 
 // This class represents a read-only file that lives in an S3-style blob store.  It reads using the REST API.
-class AsyncFileS3BlobStoreRead : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreRead> {
+class AsyncFileS3BlobStoreRead final : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreRead> {
 public:
-	virtual void addref() { ReferenceCounted<AsyncFileS3BlobStoreRead>::addref(); }
+	void addref() override { ReferenceCounted<AsyncFileS3BlobStoreRead>::addref(); }
-	virtual void delref() { ReferenceCounted<AsyncFileS3BlobStoreRead>::delref(); }
+	void delref() override { ReferenceCounted<AsyncFileS3BlobStoreRead>::delref(); }
 
 	Future<int> read(void* data, int length, int64_t offset) override;
 
@@ -281,7 +281,7 @@ public:
 
 	std::string getFilename() const override { return m_object; }
 
-	virtual ~AsyncFileS3BlobStoreRead() {}
+	~AsyncFileS3BlobStoreRead() override {}
 
 	Reference<S3BlobStoreEndpoint> m_bstore;
 	std::string m_bucket;

fdbclient/BackupContainer.h

@@ -40,7 +40,7 @@ Future<Version> timeKeeperVersionFromDatetime(std::string const &datetime, Datab
 // TODO: Move the log file and range file format encoding/decoding stuff to this file and behind interfaces.
 class IBackupFile {
 public:
-	IBackupFile(const std::string& fileName) : m_fileName(fileName), m_offset(0) {}
+	IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
 	virtual ~IBackupFile() {}
 	// Backup files are append-only and cannot have more than 1 append outstanding at once.
 	virtual Future<Void> append(const void *data, int len) = 0;
@@ -48,16 +48,13 @@ public:
 	inline std::string getFileName() const {
 		return m_fileName;
 	}
-	inline int64_t size() const {
+	virtual int64_t size() const = 0;
-		return m_offset;
-	}
 	virtual void addref() = 0;
 	virtual void delref() = 0;
 
 	Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
 protected:
 	std::string m_fileName;
-	int64_t m_offset;
 };
 
 // Structures for various backup components

fdbclient/BackupContainerAzureBlobStore.actor.cpp

@@ -44,7 +44,7 @@ public:
 			                                  blobName = this->blobName, data, length, offset] {
 				std::ostringstream oss(std::ios::out | std::ios::binary);
 				client->download_blob_to_stream(containerName, blobName, offset, length, oss);
-				auto str = oss.str();
+				auto str = std::move(oss).str();
 				memcpy(data, str.c_str(), str.size());
 				return static_cast<int>(str.size());
 			});
@@ -127,9 +127,11 @@ public:
 
 	class BackupFile final : public IBackupFile, ReferenceCounted<BackupFile> {
 		Reference<IAsyncFile> m_file;
+		int64_t m_offset;
 
 	public:
-		BackupFile(const std::string& fileName, Reference<IAsyncFile> file) : IBackupFile(fileName), m_file(file) {}
+		BackupFile(const std::string& fileName, Reference<IAsyncFile> file)
+		  : IBackupFile(fileName), m_file(file), m_offset(0) {}
 		Future<Void> append(const void* data, int len) override {
 			Future<Void> r = m_file->write(data, len, m_offset);
 			m_offset += len;
@@ -142,6 +144,7 @@ public:
 				return Void();
 			});
 		}
+		int64_t size() const override { return m_offset; }
 		void addref() override { ReferenceCounted<BackupFile>::addref(); }
 		void delref() override { ReferenceCounted<BackupFile>::delref(); }
 	};

fdbclient/BackupContainerFileSystem.actor.cpp

@@ -159,8 +159,7 @@ public:
 		state int i;
 
 		// Validate each filename, update version range
-		for (i = 0; i < fileNames.size(); ++i) {
+		for (const auto& f : fileNames) {
-			auto const& f = fileNames[i];
 			if (pathToRangeFile(rf, f, 0)) {
 				fileArray.push_back(f);
 				if (rf.version < minVer) minVer = rf.version;

fdbclient/BackupContainerFileSystem.h

@@ -73,7 +73,7 @@ public:
 	void delref() override = 0;
 
 	BackupContainerFileSystem() {}
-	virtual ~BackupContainerFileSystem() {}
+	~BackupContainerFileSystem() override {}
 
 	// Create the container
 	Future<Void> create() override = 0;

fdbclient/BackupContainerLocalDirectory.actor.cpp

@@ -30,16 +30,38 @@ namespace {
 
 class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
 public:
-	BackupFile(std::string fileName, Reference<IAsyncFile> file, std::string finalFullPath)
+	BackupFile(const std::string& fileName, Reference<IAsyncFile> file, const std::string& finalFullPath)
-	  : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath) {}
+	  : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0) {
+		m_buffer.reserve(m_buffer.arena(), CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK);
+	}
+
+	Future<Void> append(const void* data, int len) override {
+		m_buffer.append(m_buffer.arena(), (const uint8_t*)data, len);
+
+		if (m_buffer.size() >= CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) {
+			return flush(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK);
+		}
+
+		return Void();
+	}
+
+	Future<Void> flush(int size) {
+		ASSERT(size <= m_buffer.size());
+
+		// Keep a reference to the old buffer
+		Standalone<VectorRef<uint8_t>> old = m_buffer;
+		// Make a new buffer, initialized with the excess bytes over the block size from the old buffer
+		m_buffer = Standalone<VectorRef<uint8_t>>(old.slice(size, old.size()));
+
+		// Write the old buffer to the underlying file and update the write offset
+		Future<Void> r = holdWhile(old, m_file->write(old.begin(), size, m_writeOffset));
+		m_writeOffset += size;
 
-	Future<Void> append(const void* data, int len) {
-		Future<Void> r = m_file->write(data, len, m_offset);
-		m_offset += len;
 		return r;
 	}
 
 	ACTOR static Future<Void> finish_impl(Reference<BackupFile> f) {
+		wait(f->flush(f->m_buffer.size()));
 		wait(f->m_file->truncate(f->size())); // Some IAsyncFile implementations extend in whole block sizes.
 		wait(f->m_file->sync());
 		std::string name = f->m_file->getFilename();
@@ -48,13 +70,17 @@ public:
 		return Void();
 	}
 
-	Future<Void> finish() { return finish_impl(Reference<BackupFile>::addRef(this)); }
+	int64_t size() const override { return m_buffer.size() + m_writeOffset; }
+
+	Future<Void> finish() override { return finish_impl(Reference<BackupFile>::addRef(this)); }
 
 	void addref() override { return ReferenceCounted<BackupFile>::addref(); }
 	void delref() override { return ReferenceCounted<BackupFile>::delref(); }
 
 private:
 	Reference<IAsyncFile> m_file;
+	Standalone<VectorRef<uint8_t>> m_buffer;
+	int64_t m_writeOffset;
 	std::string m_finalFullPath;
 };
 
@@ -72,7 +98,7 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
 		                   [](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }),
 		    files.end());
 
-	for (auto& f : files) {
+	for (const auto& f : files) {
 		// Hide .part or .temp files.
 		StringRef s(f);
 		if (!s.endsWith(LiteralStringRef(".part")) && !s.endsWith(LiteralStringRef(".temp")))
@@ -147,7 +173,7 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
 	std::vector<std::string> dirs = platform::listDirectories(path);
 	std::vector<std::string> results;
 
-	for (auto& r : dirs) {
+	for (const auto& r : dirs) {
 		if (r == "." || r == "..") continue;
 		results.push_back(std::string("file://") + joinPath(path, r));
 	}

fdbclient/BackupContainerS3BlobStore.actor.cpp

@@ -36,7 +36,7 @@ public:
 		state std::string basePath = INDEXFOLDER + '/';
 		S3BlobStoreEndpoint::ListResult contents = wait(bstore->listObjects(bucket, basePath));
 		std::vector<std::string> results;
-		for (auto& f : contents.objects) {
+		for (const auto& f : contents.objects) {
 			results.push_back(
 			    bstore->getResourceURL(f.name.substr(basePath.size()), format("bucket=%s", bucket.c_str())));
 		}
@@ -45,15 +45,16 @@ public:
 
 	class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
 	public:
-		BackupFile(std::string fileName, Reference<IAsyncFile> file) : IBackupFile(fileName), m_file(file) {}
+		BackupFile(std::string fileName, Reference<IAsyncFile> file)
+		  : IBackupFile(fileName), m_file(file), m_offset(0) {}
 
-		Future<Void> append(const void* data, int len) {
+		Future<Void> append(const void* data, int len) override {
 			Future<Void> r = m_file->write(data, len, m_offset);
 			m_offset += len;
 			return r;
 		}
 
-		Future<Void> finish() {
+		Future<Void> finish() override {
 			Reference<BackupFile> self = Reference<BackupFile>::addRef(this);
 			return map(m_file->sync(), [=](Void _) {
 				self->m_file.clear();
@@ -61,11 +62,14 @@ public:
 			});
 		}
 
+		int64_t size() const override { return m_offset; }
+
 		void addref() final { return ReferenceCounted<BackupFile>::addref(); }
 		void delref() final { return ReferenceCounted<BackupFile>::delref(); }
 
 	private:
 		Reference<IAsyncFile> m_file;
+		int64_t m_offset;
 	};
 
 	ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
@@ -82,7 +86,7 @@ public:
 		state S3BlobStoreEndpoint::ListResult result = wait(bc->m_bstore->listObjects(
 		    bc->m_bucket, bc->dataPath(path), '/', std::numeric_limits<int>::max(), rawPathFilter));
 		BackupContainerFileSystem::FilesAndSizesT files;
-		for (auto& o : result.objects) {
+		for (const auto& o : result.objects) {
 			ASSERT(o.name.size() >= prefixTrim);
 			files.push_back({ o.name.substr(prefixTrim), o.size });
 		}
@@ -135,15 +139,13 @@ BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndp
   : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
 
 	// Currently only one parameter is supported, "bucket"
-	for (auto& kv : params) {
+	for (const auto& [name, value] : params) {
-		if (kv.first == "bucket") {
+		if (name == "bucket") {
-			m_bucket = kv.second;
+			m_bucket = value;
 			continue;
 		}
-		TraceEvent(SevWarn, "BackupContainerS3BlobStoreInvalidParameter")
+		TraceEvent(SevWarn, "BackupContainerS3BlobStoreInvalidParameter").detail("Name", name).detail("Value", value);
-		    .detail("Name", kv.first)
+		IBackupContainer::lastOpenError = format("Unknown URL parameter: '%s'", name.c_str());
-		    .detail("Value", kv.second);
-		IBackupContainer::lastOpenError = format("Unknown URL parameter: '%s'", kv.first.c_str());
 		throw backup_invalid_url();
 	}
 }

fdbclient/BuildFlags.h.in

@@ -0,0 +1,80 @@
+/*
+ * BuildFlags.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_BUILDFLAGS_H
+#define FDBCLIENT_BUILDFLAGS_H
+
+#include <string>
+
+#include "fdbclient/JSONDoc.h"
+
+#ifdef __GLIBC__
+#define C_VERSION_MAJOR __GLIBC__
+#define C_VERSION_MINOR __GLIBC_MINOR__
+#else
+#define C_VERSION_MAJOR 0
+#define C_VERSION_MINOR 0
+#endif
+
+// FDB info.
+const std::string kGitHash = "@CURRENT_GIT_VERSION_WNL@";
+const std::string kFdbVersion = "@FDB_VERSION@";
+
+// System architecture.
+const std::string kArch = "@CMAKE_SYSTEM@";
+// ID of compiler used for build, ie "Clang", "GNU", etc...
+const std::string kCompiler = "@CMAKE_CXX_COMPILER_ID@";
+
+// Library versions.
+const std::string kBoostVersion = "@Boost_LIB_VERSION@";
+
+// Build info and flags.
+const std::string kCMakeVersion = "@CMAKE_VERSION@";
+const std::string kCCacheEnabled = "@USE_CCACHE@";
+
+// GNU C library major and minor versions.
+constexpr int kCVersionMajor = C_VERSION_MAJOR;
+constexpr int kCVersionMinor = C_VERSION_MINOR;
+
+// C++ standard. Possible values are 201402L, 201703L, etc...
+constexpr int kCppStandard = __cplusplus;
+
+// Returns a JSON string with information about how the binary was built.
+std::string jsonBuildInformation() {
+	json_spirit::mValue json;	
+	JSONDoc doc(json);
+
+	doc.create("git_hash") = kGitHash;
+	doc.create("fdb_version") = kFdbVersion;
+
+	doc.create("arch") = kArch;
+	doc.create("compiler") = kCompiler;
+
+	doc.create("boost_version") = kBoostVersion;
+	doc.create("cmake_version") = kCMakeVersion;
+	doc.create("ccache") = kCCacheEnabled;
+
+	doc.create("glibc_version") = std::to_string(kCVersionMajor) + "." + std::to_string(kCVersionMinor);
+	doc.create("cpp_standard") = kCppStandard;
+
+	return json_spirit::write_string(json, json_spirit::pretty_print) + "\n";
+}
+
+#endif

fdbclient/ClientLogEvents.h

@@ -23,46 +23,39 @@
 #define FDBCLIENT_CLIENTLOGEVENTS_H
 
 namespace FdbClientLogEvents {
-	typedef int EventType;
+enum class EventType {
-	enum {	GET_VERSION_LATENCY	= 0,
+	GET_VERSION_LATENCY = 0,
-			GET_LATENCY			= 1,
+	GET_LATENCY = 1,
-			GET_RANGE_LATENCY	= 2,
+	GET_RANGE_LATENCY = 2,
-			COMMIT_LATENCY		= 3,
+	COMMIT_LATENCY = 3,
-			ERROR_GET			= 4,
+	ERROR_GET = 4,
-			ERROR_GET_RANGE		= 5,
+	ERROR_GET_RANGE = 5,
-			ERROR_COMMIT		= 6,
+	ERROR_COMMIT = 6,
+	UNSET
+};
 
-			EVENTTYPEEND	// End of EventType
+enum class TransactionPriorityType { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, PRIORITY_IMMEDIATE = 2, UNSET };
-	     };
 
-	typedef int TrasactionPriorityType;
+struct Event {
-	enum {
+	Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
-		PRIORITY_DEFAULT   = 0,
+		if (dc.present()) dcId = dc.get();
-		PRIORITY_BATCH     = 1,
+	}
-		PRIORITY_IMMEDIATE = 2,
+	Event() {}
-		PRIORITY_END
-	};
 
-	struct Event {
+	template <typename Ar>
-		Event(EventType t, double ts, const Optional<Standalone<StringRef>> &dc) : type(t), startTs(ts){
+	Ar& serialize(Ar& ar) {
-			if (dc.present())
+		if (ar.protocolVersion().version() >= (uint64_t)0x0FDB00B063010001LL) {
-				dcId = dc.get(); 
+			return serializer(ar, type, startTs, dcId);
-		 }
+		} else {
-		Event() { }
+			return serializer(ar, type, startTs);
-
-		template <typename Ar>	Ar& serialize(Ar &ar) { 
-			if (ar.protocolVersion().version() >= (uint64_t) 0x0FDB00B063010001LL) {
-				return serializer(ar, type, startTs, dcId);
-			} else {
-				return serializer(ar, type, startTs); 
-			}
 		}
+	}
 
-		EventType type{ EVENTTYPEEND };
+	EventType type{ EventType::UNSET };
-		double startTs{ 0 };
+	double startTs{ 0 };
-		Key dcId{};
+	Key dcId{};
 
-		void logEvent(std::string id, int maxFieldLength) const {}
+	void logEvent(std::string id, int maxFieldLength) const {}
 	};
 
 	struct EventGetVersion : public Event {
@@ -96,9 +89,9 @@ namespace FdbClientLogEvents {
 		}
 
 		double latency;
-		TrasactionPriorityType priorityType {PRIORITY_END};
+	    TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
 
-		void logEvent(std::string id, int maxFieldLength) const {
+	    void logEvent(std::string id, int maxFieldLength) const {
 			TraceEvent("TransactionTrace_GetVersion")
 			.detail("TransactionID", id)
 			.detail("Latency", latency)
@@ -108,23 +101,25 @@ namespace FdbClientLogEvents {
 
 	// Version V3 of EventGetVersion starting at 6.3
 	struct EventGetVersion_V3 : public Event {
-		EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, TransactionPriority priority, Version version) : Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
+	    EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>>& dcId, double lat,
-			switch(priority) {
+	                       TransactionPriority priority, Version version)
+	      : Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
+		    switch(priority) {
 				// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority
 				case TransactionPriority::IMMEDIATE:
-					priorityType = PRIORITY_IMMEDIATE;
+			        priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
-					break;
+			        break;
 				case TransactionPriority::DEFAULT:
-					priorityType = PRIORITY_DEFAULT;
+			        priorityType = TransactionPriorityType::PRIORITY_DEFAULT;
-					break;
+			        break;
 				case TransactionPriority::BATCH:
-					priorityType = PRIORITY_BATCH;
+			        priorityType = TransactionPriorityType::PRIORITY_BATCH;
-					break;
+			        break;
 				default:
 					ASSERT(false);
 			}
-		}
+	    }
-		EventGetVersion_V3() { }
+	    EventGetVersion_V3() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -134,8 +129,8 @@ namespace FdbClientLogEvents {
 		}
 
 		double latency;
-		TrasactionPriorityType priorityType {PRIORITY_END};
+	    TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
-		Version readVersion;
+	    Version readVersion;
 
 		void logEvent(std::string id, int maxFieldLength) const {
 			TraceEvent("TransactionTrace_GetVersion")
@@ -147,8 +142,9 @@ namespace FdbClientLogEvents {
 	};
 
 	struct EventGet : public Event {
-		EventGet(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &in_key) : Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) { }
+	    EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
-		EventGet() { }
+	      : Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
+	    EventGet() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -173,8 +169,11 @@ namespace FdbClientLogEvents {
 	};
 
 	struct EventGetRange : public Event {
-		EventGetRange(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &start_key, const KeyRef & end_key) : Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) { }
+	    EventGetRange(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size,
-		EventGetRange() { }
+	                  const KeyRef& start_key, const KeyRef& end_key)
+	      : Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
+	        endKey(end_key) {}
+	    EventGetRange() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -252,9 +251,11 @@ namespace FdbClientLogEvents {
 
 	// Version V2 of EventGetVersion starting at 6.3
 	struct EventCommit_V2 : public Event {
-		EventCommit_V2(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int mut, int bytes, Version version, const CommitTransactionRequest &commit_req) 
+	    EventCommit_V2(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int mut, int bytes,
-			: Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version), req(commit_req) { }
+	                   Version version, const CommitTransactionRequest& commit_req)
-		EventCommit_V2() { }
+	      : Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes),
+	        commitVersion(version), req(commit_req) {}
+	    EventCommit_V2() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -306,8 +307,9 @@ namespace FdbClientLogEvents {
 	};
 
 	struct EventGetError : public Event {
-		EventGetError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &in_key) : Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) { }
+	    EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
-		EventGetError() { }
+	      : Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
+	    EventGetError() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -330,8 +332,10 @@ namespace FdbClientLogEvents {
 	};
 
 	struct EventGetRangeError : public Event {
-		EventGetRangeError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &start_key, const KeyRef & end_key) : Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) { }
+	    EventGetRangeError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
-		EventGetRangeError() { }
+	                       const KeyRef& start_key, const KeyRef& end_key)
+	      : Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
+	    EventGetRangeError() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)
@@ -356,8 +360,10 @@ namespace FdbClientLogEvents {
 	};
 
 	struct EventCommitError : public Event {
-		EventCommitError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const CommitTransactionRequest &commit_req) : Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) { }
+	    EventCommitError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
-		EventCommitError() { }
+	                     const CommitTransactionRequest& commit_req)
+	      : Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
+	    EventCommitError() { }
 
 		template <typename Ar>	Ar& serialize(Ar &ar) {
 			if (!ar.isDeserializing)

fdbclient/ClientWorkerInterface.h

@@ -93,7 +93,5 @@ struct ProfilerRequest {
 		serializer(ar, reply, type, action, duration, outputFile);
 	}
 };
-BINARY_SERIALIZABLE( ProfilerRequest::Type );
-BINARY_SERIALIZABLE( ProfilerRequest::Action );
 
 #endif

fdbclient/DatabaseBackupAgent.actor.cpp

@@ -138,10 +138,10 @@ namespace dbBackup {
 		static const Key keyAddBackupRangeTasks;
 		static const Key keyBackupRangeBeginKey;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		ACTOR static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr, Key beginKey, Key endKey, int limit) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -459,10 +459,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 	};
 	StringRef FinishFullBackupTaskFunc::name = LiteralStringRef("dr_finish_full_backup");
@@ -472,10 +472,10 @@ namespace dbBackup {
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
 
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state FlowLock lock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
@@ -533,10 +533,10 @@ namespace dbBackup {
 
 		static const Key keyNextBeginVersion;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		// store mutation data from results until the end of stream or the timeout. If breaks on timeout returns the first uncopied version
 		ACTOR static Future<Optional<Version>> dumpData(
@@ -866,10 +866,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyLogsTaskFunc::name = LiteralStringRef("dr_copy_logs");
 	REGISTER_TASKFUNC(CopyLogsTaskFunc);
@@ -879,7 +879,7 @@ namespace dbBackup {
 	    static constexpr uint32_t version = 1;
 	    static const Key keyInsertTask;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
 		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
@@ -968,8 +968,8 @@ namespace dbBackup {
 			return Void();
 		}
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("dr_finished_full_backup");
 	const Key FinishedFullBackupTaskFunc::keyInsertTask = LiteralStringRef("insertTask");
@@ -1048,10 +1048,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyDiffLogsTaskFunc::name = LiteralStringRef("dr_copy_diff_logs");
 	REGISTER_TASKFUNC(CopyDiffLogsTaskFunc);
@@ -1067,10 +1067,10 @@ namespace dbBackup {
 			return Void();
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef SkipOldEraseLogRangeTaskFunc::name = LiteralStringRef("dr_skip_legacy_task");
 	REGISTER_TASKFUNC(SkipOldEraseLogRangeTaskFunc);
@@ -1087,10 +1087,10 @@ namespace dbBackup {
 
 		static const Key keyNextBeginVersion;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		ACTOR static Future<Void> dumpData(Database cx, Reference<Task> task, PromiseStream<RCGroup> results, FlowLock* lock, Reference<TaskBucket> tb) {
 			state bool endOfStream = false;
@@ -1303,10 +1303,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortOldBackupTaskFunc::name = LiteralStringRef("dr_abort_legacy_backup");
 	REGISTER_TASKFUNC(AbortOldBackupTaskFunc);
@@ -1421,10 +1421,10 @@ namespace dbBackup {
 			return Void();
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyDiffLogsUpgradeTaskFunc::name = LiteralStringRef("db_copy_diff_logs");
 	REGISTER_TASKFUNC(CopyDiffLogsUpgradeTaskFunc);
@@ -1512,10 +1512,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupRestorableTaskFunc::name = LiteralStringRef("dr_backup_restorable");
 	REGISTER_TASKFUNC(BackupRestorableTaskFunc);
@@ -1711,10 +1711,10 @@ namespace dbBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("dr_start_full_backup");
 	REGISTER_TASKFUNC(StartFullBackupTaskFunc);

fdbclient/DatabaseConfiguration.h

@@ -51,9 +51,19 @@ struct RegionInfo {
 	int32_t priority;
 
 	Reference<IReplicationPolicy> satelliteTLogPolicy;
+
+	// Number of tLogs that should be recruited in satellite datacenters.
 	int32_t satelliteDesiredTLogCount;
+
+	// Total number of copies made for each mutation across all satellite tLogs in all DCs.
 	int32_t satelliteTLogReplicationFactor;
+
+	// Number of tLog replies we can ignore when waiting for quorum. Hence, effective quorum is
+	// satelliteDesiredTLogCount -  satelliteTLogWriteAntiQuorum. Locality of individual tLogs is not taken
+	// into account.
 	int32_t satelliteTLogWriteAntiQuorum;
+
+	// Number of satellite datacenters for current region, as set by `satellite_redundancy_mode`.
 	int32_t satelliteTLogUsableDcs;
 
 	Reference<IReplicationPolicy> satelliteTLogPolicyFallback;
@@ -63,27 +73,32 @@ struct RegionInfo {
 
 	std::vector<SatelliteInfo> satellites;
 
-	RegionInfo() : priority(0), satelliteDesiredTLogCount(-1), satelliteTLogReplicationFactor(0), satelliteTLogWriteAntiQuorum(0), satelliteTLogUsableDcs(1),
+	RegionInfo()
-		satelliteTLogReplicationFactorFallback(0), satelliteTLogWriteAntiQuorumFallback(0), satelliteTLogUsableDcsFallback(0) {}
+	  : priority(0), satelliteDesiredTLogCount(-1), satelliteTLogReplicationFactor(0), satelliteTLogWriteAntiQuorum(0),
+	    satelliteTLogUsableDcs(1), satelliteTLogReplicationFactorFallback(0), satelliteTLogWriteAntiQuorumFallback(0),
+	    satelliteTLogUsableDcsFallback(0) {}
 
 	struct sort_by_priority {
-		bool operator ()(RegionInfo const&a, RegionInfo const& b) const { return a.priority > b.priority; }
+		bool operator()(RegionInfo const& a, RegionInfo const& b) const { return a.priority > b.priority; }
 	};
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor, satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs,
+		serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor,
-			satelliteTLogPolicyFallback, satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback, satelliteTLogUsableDcsFallback, satellites);
+		           satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback,
+		           satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback,
+		           satelliteTLogUsableDcsFallback, satellites);
 	}
 };
 
 struct DatabaseConfiguration {
 	DatabaseConfiguration();
 
-	void applyMutation( MutationRef mutation );
+	void applyMutation(MutationRef mutation);
-	bool set( KeyRef key, ValueRef value );  // Returns true if a configuration option that requires recovery to take effect is changed
+	bool set(KeyRef key,
-	bool clear( KeyRangeRef keys );
+	         ValueRef value); // Returns true if a configuration option that requires recovery to take effect is changed
-	Optional<ValueRef> get( KeyRef key ) const;
+	bool clear(KeyRangeRef keys);
+	Optional<ValueRef> get(KeyRef key) const;
 
 	bool isValid() const;
 
@@ -92,63 +107,75 @@ struct DatabaseConfiguration {
 	std::string toString() const;
 	StatusObject toJSON(bool noPolicies = false) const;
 	StatusArray getRegionJSON() const;
-	
+
-	RegionInfo getRegion( Optional<Key> dcId ) const {
+	RegionInfo getRegion(Optional<Key> dcId) const {
-		if(!dcId.present()) {
+		if (!dcId.present()) {
 			return RegionInfo();
 		}
-		for(auto& r : regions) {
+		for (auto& r : regions) {
-			if(r.dcId == dcId.get()) {
+			if (r.dcId == dcId.get()) {
 				return r;
 			}
 		}
 		return RegionInfo();
 	}
 
-	int expectedLogSets( Optional<Key> dcId ) const {
+	int expectedLogSets(Optional<Key> dcId) const {
 		int result = 1;
-		if(dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0 && usableRegions > 1) {
+		if (dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0 && usableRegions > 1) {
 			result++;
 		}
-		
+
-		if(usableRegions > 1) {
+		if (usableRegions > 1) {
 			result++;
 		}
 		return result;
 	}
 
+	// Counts the number of DCs required including remote and satellites for current database configuraiton.
 	int32_t minDatacentersRequired() const {
 		int minRequired = 0;
-		for(auto& r : regions) {
+		for (auto& r : regions) {
 			minRequired += 1 + r.satellites.size();
 		}
 		return minRequired;
 	}
+
 	int32_t minZonesRequiredPerDatacenter() const {
-		int minRequired = std::max( remoteTLogReplicationFactor, std::max(tLogReplicationFactor, storageTeamSize) );
+		int minRequired = std::max(remoteTLogReplicationFactor, std::max(tLogReplicationFactor, storageTeamSize));
-		for(auto& r : regions) {
+		for (auto& r : regions) {
-			minRequired = std::max( minRequired, r.satelliteTLogReplicationFactor/std::max(1, r.satelliteTLogUsableDcs) );
+			minRequired =
+			    std::max(minRequired, r.satelliteTLogReplicationFactor / std::max(1, r.satelliteTLogUsableDcs));
 		}
 		return minRequired;
 	}
 
-	//Killing an entire datacenter counts as killing one zone in modes that support it
+	// Retuns the maximum number of discrete failures a cluster can tolerate.
+	// In HA mode, `fullyReplicatedRegions` is set to false initially when data is being
+	// replicated to remote, and will be true later. `forAvailablity` is set to true
+	// if we want to account the number for machines that can recruit new tLogs/SS after failures.
+	// Killing an entire datacenter counts as killing one zone in modes that support it
 	int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const {
 		int worstSatellite = regions.size() ? std::numeric_limits<int>::max() : 0;
 		int regionsWithNonNegativePriority = 0;
-		for(auto& r : regions) {
+		for (auto& r : regions) {
-			if(r.priority >= 0) {
+			if (r.priority >= 0) {
 				regionsWithNonNegativePriority++;
 			}
-			worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
+			worstSatellite =
-			if(r.satelliteTLogUsableDcsFallback > 0) {
+			    std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
-				worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
+			if (r.satelliteTLogUsableDcsFallback > 0) {
+				worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback -
+				                                              r.satelliteTLogWriteAntiQuorumFallback);
 			}
 		}
-		if(usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && (!forAvailability || regionsWithNonNegativePriority > 1)) {
+		if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
-			return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1), storageTeamSize - 1);
+		    (!forAvailability || regionsWithNonNegativePriority > 1)) {
-		} else if(worstSatellite > 0) {
+			return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1),
-			return std::min(tLogReplicationFactor + worstSatellite - 2 - tLogWriteAntiQuorum, storageTeamSize - 1);
+			                    storageTeamSize - 1);
+		} else if (worstSatellite > 0) {
+			// Primary and Satellite tLogs are synchronously replicated, hence we can lose all but 1.
+			return std::min(tLogReplicationFactor + worstSatellite - 1 - tLogWriteAntiQuorum, storageTeamSize - 1);
 		}
 		return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1);
 	}
@@ -187,13 +214,13 @@ struct DatabaseConfiguration {
 	// Backup Workers
 	bool backupWorkerEnabled;
 
-	//Data centers
+	// Data centers
-	int32_t usableRegions;
+	int32_t usableRegions;  // Number of regions which have a replica of the database.
 	int32_t repopulateRegionAntiQuorum;
 	std::vector<RegionInfo> regions;
 
 	// Excluded servers (no state should be here)
-	bool isExcludedServer( NetworkAddressList ) const;
+	bool isExcludedServer(NetworkAddressList) const;
 	std::set<AddressExclusion> getExcludedServers() const;
 
 	int32_t getDesiredCommitProxies() const {
@@ -204,17 +231,33 @@ struct DatabaseConfiguration {
 		if (grvProxyCount == -1) return autoGrvProxyCount;
 		return grvProxyCount;
 	}
-	int32_t getDesiredResolvers() const { if(resolverCount == -1) return autoResolverCount; return resolverCount; }
+	int32_t getDesiredResolvers() const {
-	int32_t getDesiredLogs() const { if(desiredTLogCount == -1) return autoDesiredTLogCount; return desiredTLogCount; }
+		if (resolverCount == -1) return autoResolverCount;
-	int32_t getDesiredRemoteLogs() const { if(remoteDesiredTLogCount == -1) return getDesiredLogs(); return remoteDesiredTLogCount;  }
+		return resolverCount;
-	int32_t getDesiredSatelliteLogs( Optional<Key> dcId ) const {
+	}
-		auto desired = getRegion(dcId).satelliteDesiredTLogCount;
+	int32_t getDesiredLogs() const {
-		if(desired == -1) return autoDesiredTLogCount; return desired;
+		if (desiredTLogCount == -1) return autoDesiredTLogCount;
+		return desiredTLogCount;
+	}
+	int32_t getDesiredRemoteLogs() const {
+		if (remoteDesiredTLogCount == -1) return getDesiredLogs();
+		return remoteDesiredTLogCount;
+	}
+	int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
+		auto desired = getRegion(dcId).satelliteDesiredTLogCount;
+		if (desired == -1) return autoDesiredTLogCount;
+		return desired;
+	}
+	int32_t getRemoteTLogReplicationFactor() const {
+		if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor;
+		return remoteTLogReplicationFactor;
+	}
+	Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
+		if (remoteTLogReplicationFactor == 0) return tLogPolicy;
+		return remoteTLogPolicy;
 	}
-	int32_t getRemoteTLogReplicationFactor() const { if(remoteTLogReplicationFactor == 0) return tLogReplicationFactor; return remoteTLogReplicationFactor; }
-	Reference<IReplicationPolicy> getRemoteTLogPolicy() const { if(remoteTLogReplicationFactor == 0) return tLogPolicy; return remoteTLogPolicy; }
 
-	bool operator == ( DatabaseConfiguration const& rhs ) const {
+	bool operator==(DatabaseConfiguration const& rhs) const {
 		const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
 		const_cast<DatabaseConfiguration*>(&rhs)->makeConfigurationImmutable();
 		return rawConfiguration == rhs.rawConfiguration;
@@ -226,8 +269,7 @@ struct DatabaseConfiguration {
 		if (!ar.isDeserializing) makeConfigurationImmutable();
 		serializer(ar, rawConfiguration);
 		if (ar.isDeserializing) {
-			for(auto c=rawConfiguration.begin(); c!=rawConfiguration.end(); ++c)
+			for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value);
-				setInternal(c->key, c->value);
 			setDefaultReplicationPolicy();
 		}
 	}
@@ -235,13 +277,13 @@ struct DatabaseConfiguration {
 	void fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig);
 
 private:
-	Optional< std::map<std::string, std::string> > mutableConfiguration;  // If present, rawConfiguration is not valid
+	Optional<std::map<std::string, std::string>> mutableConfiguration; // If present, rawConfiguration is not valid
-	Standalone<VectorRef<KeyValueRef>> rawConfiguration;   // sorted by key
+	Standalone<VectorRef<KeyValueRef>> rawConfiguration; // sorted by key
 
 	void makeConfigurationMutable();
 	void makeConfigurationImmutable();
 
-	bool setInternal( KeyRef key, ValueRef value );
+	bool setInternal(KeyRef key, ValueRef value);
 	void resetInternal();
 	void setDefaultReplicationPolicy();
 

fdbclient/DatabaseContext.h

@@ -43,7 +43,8 @@ public:
 	static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality );
 	void notifyContextDestroyed();
 
-	virtual ~StorageServerInfo();
+	~StorageServerInfo() override;
+
 private:
 	DatabaseContext *cx;
 	StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
@@ -205,6 +206,11 @@ public:
 	Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby);
 	Future<Void> connectionFileChanged();
 	bool switchable = false;
+ 
+	// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
+	Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
+	// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
+	Future<Void> forceRecoveryWithDataLoss(StringRef dcId);
 
 //private: 
 	explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
@@ -303,6 +309,7 @@ public:
 	Counter transactionsCommitCompleted;
 	Counter transactionKeyServerLocationRequests;
 	Counter transactionKeyServerLocationRequestsCompleted;
+	Counter transactionStatusRequests;
 	Counter transactionsTooOld;
 	Counter transactionsFutureVersions;
 	Counter transactionsNotCommitted;

fdbclient/FDBTypes.h

@@ -71,9 +71,7 @@ struct Tag {
 	bool operator != ( const Tag& r ) const { return locality!=r.locality || id!=r.id; }
 	bool operator < ( const Tag& r ) const { return locality < r.locality || (locality == r.locality && id < r.id); }
 
-	int toTagDataIndex() {
+	int toTagDataIndex() const { return locality >= 0 ? 2 * locality : 1 - (2 * locality); }
-		return locality >= 0 ? 2 * locality : 1 - (2 * locality);
-	}
 
 	std::string toString() const {
 		return format("%d:%d", locality, id);

fdbclient/FileBackupAgent.actor.cpp

@@ -802,8 +802,8 @@ namespace fileBackup {
 			return StringRef();
 	    }
 
-	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
+	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Future<Void>(Void()); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortFiveZeroBackupTask::name = LiteralStringRef("abort_legacy_backup");
 	REGISTER_TASKFUNC(AbortFiveZeroBackupTask);
@@ -872,8 +872,8 @@ namespace fileBackup {
 			return StringRef();
 	    }
 
-	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
+	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Future<Void>(Void()); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortFiveOneBackupTask::name = LiteralStringRef("abort_legacy_backup_5.2");
 	REGISTER_TASKFUNC(AbortFiveOneBackupTask);
@@ -995,10 +995,10 @@ namespace fileBackup {
 			);
 	    }
 
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable.
 		// This means:
@@ -1289,10 +1289,10 @@ namespace fileBackup {
 			}
 		} Params;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, int priority, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>(), Version scheduledVersion = invalidVersion) {
 			Key key = wait(addBackupTask(name,
@@ -1809,10 +1809,10 @@ namespace fileBackup {
 			}
 		} Params;
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
@@ -1995,7 +1995,7 @@ namespace fileBackup {
 	struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
 		static struct {
 			static TaskParam<Version> beginVersion() {
@@ -2041,8 +2041,8 @@ namespace fileBackup {
 			return Void();
 		}
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("file_backup_erase_logs_5.2");
 	REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
@@ -2166,10 +2166,10 @@ namespace fileBackup {
 			return key;
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs_5.2");
 	REGISTER_TASKFUNC(BackupLogsDispatchTask);
@@ -2178,7 +2178,7 @@ namespace fileBackup {
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
 
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
 		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version));
@@ -2208,8 +2208,8 @@ namespace fileBackup {
 			return key;
 		}
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished_5.2");
 	REGISTER_TASKFUNC(FileBackupFinishedTask);
@@ -2366,10 +2366,10 @@ namespace fileBackup {
 			return key;
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest_5.2");
 	REGISTER_TASKFUNC(BackupSnapshotManifest);
@@ -2518,10 +2518,10 @@ namespace fileBackup {
 			return key;
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start_5.2");
 	REGISTER_TASKFUNC(StartFullBackupTaskFunc);
@@ -2567,10 +2567,10 @@ namespace fileBackup {
 
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 
 	};
 	StringRef RestoreCompleteTaskFunc::name = LiteralStringRef("restore_complete");
@@ -2830,10 +2830,10 @@ namespace fileBackup {
 
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreRangeTaskFunc::name = LiteralStringRef("restore_range_data");
 	REGISTER_TASKFUNC(RestoreRangeTaskFunc);
@@ -2841,7 +2841,7 @@ namespace fileBackup {
 	struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
 		static struct : InputParams {
 		} Params;
@@ -2979,8 +2979,8 @@ namespace fileBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreLogDataTaskFunc::name = LiteralStringRef("restore_log_data");
 	REGISTER_TASKFUNC(RestoreLogDataTaskFunc);
@@ -2988,7 +2988,7 @@ namespace fileBackup {
 	struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		static StringRef name;
 	    static constexpr uint32_t version = 1;
-	    StringRef getName() const { return name; };
+	    StringRef getName() const override { return name; };
 
 		static struct {
 			static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
@@ -3291,8 +3291,8 @@ namespace fileBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreDispatchTaskFunc::name = LiteralStringRef("restore_dispatch");
 	REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
@@ -3577,10 +3577,10 @@ namespace fileBackup {
 			return LiteralStringRef("OnSetAddTask");
 		}
 
-		StringRef getName() const { return name; };
+		StringRef getName() const override { return name; };
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
-		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullRestoreTaskFunc::name = LiteralStringRef("restore_start");
 	REGISTER_TASKFUNC(StartFullRestoreTaskFunc);

fdbclient/IClientApi.h

@@ -20,6 +20,7 @@
 
 #ifndef FDBCLIENT_ICLIENTAPI_H
 #define FDBCLIENT_ICLIENTAPI_H
+#include "fdbclient/ManagementAPI.actor.h"
 #pragma once
 
 #include "fdbclient/FDBOptions.g.h"
@@ -84,6 +85,11 @@ public:
 
 	virtual void addref() = 0;
 	virtual void delref() = 0;
+
+	// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
+	virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
+	// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
+	virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
 };
 
 class IClientApi {

fdbclient/Knobs.cpp

@@ -23,7 +23,8 @@
 #include "fdbclient/SystemData.h"
 #include "flow/UnitTest.h"
 
-ClientKnobs const* CLIENT_KNOBS = new ClientKnobs();
+std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
+ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
 
 #define init( knob, value ) initKnob( knob, value, #knob )
 
@@ -123,6 +124,7 @@ void ClientKnobs::initialize(bool randomize) {
 	init( TASKBUCKET_MAX_TASK_KEYS,               1000 ); if( randomize && BUGGIFY ) TASKBUCKET_MAX_TASK_KEYS = 20;
 
 	//Backup
+	init( BACKUP_LOCAL_FILE_WRITE_BLOCK,     1024*1024 ); if( randomize && BUGGIFY ) BACKUP_LOCAL_FILE_WRITE_BLOCK = 100;
 	init( BACKUP_CONCURRENT_DELETES,               100 );
 	init( BACKUP_SIMULATED_LIMIT_BYTES,		       1e6 ); if( randomize && BUGGIFY ) BACKUP_SIMULATED_LIMIT_BYTES = 1000;
 	init( BACKUP_GET_RANGE_LIMIT_BYTES,		       1e6 );

fdbclient/Knobs.h

@@ -120,6 +120,7 @@ public:
 	int TASKBUCKET_MAX_TASK_KEYS;
 
 	// Backup
+	int BACKUP_LOCAL_FILE_WRITE_BLOCK;
 	int BACKUP_CONCURRENT_DELETES;
 	int BACKUP_SIMULATED_LIMIT_BYTES;
 	int BACKUP_GET_RANGE_LIMIT_BYTES;
@@ -231,6 +232,7 @@ public:
 	void initialize(bool randomize = false);
 };
 
+extern std::unique_ptr<ClientKnobs> globalClientKnobs;
 extern ClientKnobs const* CLIENT_KNOBS;
 
 #endif

fdbclient/MultiVersionAssignmentVars.h

@@ -132,7 +132,7 @@ public:
 		api->futureSetCallback(f, &futureCallback, this);
 	}
 
-	~DLThreadSingleAssignmentVar() {
+	~DLThreadSingleAssignmentVar() override {
 		lock.assertNotEntered();
 		if(f) {
 			ASSERT_ABORT(futureRefCount == 1);

fdbclient/MultiVersionTransaction.actor.cpp

@@ -284,6 +284,31 @@ Reference<ITransaction> DLDatabase::createTransaction() {
 void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
 	throwIfError(api->databaseSetOption(db, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0));
 }
+
+ThreadFuture<int64_t> DLDatabase::rebootWorker(const StringRef& address, bool check, int duration) {
+	if(!api->databaseRebootWorker) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture *f = api->databaseRebootWorker(db, address.begin(), address.size(), check, duration);
+	return toThreadFuture<int64_t>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
+		int64_t res;
+		FdbCApi::fdb_error_t error = api->futureGetInt64(f, &res);
+		ASSERT(!error);
+		return res;
+	});
+}
+
+ThreadFuture<Void> DLDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
+	if(!api->databaseForceRecoveryWithDataLoss) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture *f = api->databaseForceRecoveryWithDataLoss(db, dcid.begin(), dcid.size());
+	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
+		return Void();
+	});
+}
 	
 // DLApi
 template<class T>
@@ -319,6 +344,8 @@ void DLApi::init() {
 	loadClientFunction(&api->databaseCreateTransaction, lib, fdbCPath, "fdb_database_create_transaction");
 	loadClientFunction(&api->databaseSetOption, lib, fdbCPath, "fdb_database_set_option");
 	loadClientFunction(&api->databaseDestroy, lib, fdbCPath, "fdb_database_destroy");
+	loadClientFunction(&api->databaseRebootWorker, lib, fdbCPath, "fdb_database_reboot_worker", headerVersion >= 700);
+	loadClientFunction(&api->databaseForceRecoveryWithDataLoss, lib, fdbCPath, "fdb_database_force_recovery_with_data_loss", headerVersion >= 700);
 
 	loadClientFunction(&api->transactionSetOption, lib, fdbCPath, "fdb_transaction_set_option");
 	loadClientFunction(&api->transactionDestroy, lib, fdbCPath, "fdb_transaction_destroy");
@@ -781,6 +808,18 @@ void MultiVersionDatabase::setOption(FDBDatabaseOptions::Option option, Optional
 	}
 }
 
+ThreadFuture<int64_t> MultiVersionDatabase::rebootWorker(const StringRef& address, bool check, int duration) {
+	if (dbState->db) {
+		return dbState->db->rebootWorker(address, check, duration);
+	}
+	return false;
+}
+
+ThreadFuture<Void> MultiVersionDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
+	auto f = dbState->db ? dbState->db->forceRecoveryWithDataLoss(dcid) : ThreadFuture<Void>(Never());
+	return abortableFuture(f, dbState->dbVar->get().onChange);
+}
+
 void MultiVersionDatabase::Connector::connect() {
 	addref();
 	onMainThreadVoid([this]() {

fdbclient/MultiVersionTransaction.h

@@ -65,7 +65,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	//Database
 	fdb_error_t (*databaseCreateTransaction)(FDBDatabase *database, FDBTransaction **tr);
 	fdb_error_t (*databaseSetOption)(FDBDatabase *database, FDBDatabaseOptions::Option option, uint8_t const *value, int valueLength);
-	void (*databaseDestroy)(FDBDatabase *database);	
+	void (*databaseDestroy)(FDBDatabase *database);
+	FDBFuture* (*databaseRebootWorker)(FDBDatabase *database, uint8_t const *address, int addressLength, fdb_bool_t check, int duration);
+	FDBFuture* (*databaseForceRecoveryWithDataLoss)(FDBDatabase *database, uint8_t const *dcid, int dcidLength);
 
 	//Transaction
 	fdb_error_t (*transactionSetOption)(FDBTransaction *tr, FDBTransactionOptions::Option option, uint8_t const *value, int valueLength);
@@ -109,6 +111,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	fdb_error_t (*futureGetDatabase)(FDBFuture *f, FDBDatabase **outDb);
 	fdb_error_t (*futureGetInt64)(FDBFuture *f, int64_t *outValue);
 	fdb_error_t (*futureGetUInt64)(FDBFuture *f, uint64_t *outValue);
+	fdb_error_t (*futureGetBool) (FDBFuture *f, bool *outValue);
 	fdb_error_t (*futureGetError)(FDBFuture *f);
 	fdb_error_t (*futureGetKey)(FDBFuture *f, uint8_t const **outKey, int *outKeyLength);
 	fdb_error_t (*futureGetValue)(FDBFuture *f, fdb_bool_t *outPresent, uint8_t const **outValue, int *outValueLength);
@@ -129,7 +132,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 class DLTransaction : public ITransaction, ThreadSafeReferenceCounted<DLTransaction> {
 public:
 	DLTransaction(Reference<FdbCApi> api, FdbCApi::FDBTransaction *tr) : api(api), tr(tr) {}
-	~DLTransaction() { api->transactionDestroy(tr); }
+	~DLTransaction() override { api->transactionDestroy(tr); }
 
 	void cancel() override;
 	void setVersion(Version v) override;
@@ -180,7 +183,7 @@ class DLDatabase : public IDatabase, ThreadSafeReferenceCounted<DLDatabase> {
 public:
 	DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase *db) : api(api), db(db), ready(Void()) {}
 	DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture);
-	~DLDatabase() {
+	~DLDatabase() override {
 		if (db) {
 			api->databaseDestroy(db);
 		}
@@ -194,6 +197,9 @@ public:
 	void addref() override { ThreadSafeReferenceCounted<DLDatabase>::addref(); }
 	void delref() override { ThreadSafeReferenceCounted<DLDatabase>::delref(); }
 
+	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
+	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
+
 private:
 	const Reference<FdbCApi> api;
 	FdbCApi::FDBDatabase* db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
@@ -315,7 +321,7 @@ class MultiVersionApi;
 class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> {
 public:
 	MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference<IDatabase> db, bool openConnectors=true);
-	~MultiVersionDatabase();
+	~MultiVersionDatabase() override;
 
 	Reference<ITransaction> createTransaction() override;
 	void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
@@ -325,6 +331,9 @@ public:
 
 	static Reference<IDatabase> debugCreateFromExistingDatabase(Reference<IDatabase> db);
 
+	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
+	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
+
 private:
 	struct DatabaseState;
 

fdbclient/NativeAPI.actor.cpp

@@ -868,13 +868,13 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
     transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc),
     transactionKeyServerLocationRequests("KeyServerLocationRequests", cc),
     transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc),
-    transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc),
+    transactionStatusRequests("StatusRequests", cc), transactionsTooOld("TooOld", cc),
-    transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc),
+    transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
-    transactionsResourceConstrained("ResourceConstrained", cc), transactionsThrottled("Throttled", cc),
+    transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
-    transactionsProcessBehind("ProcessBehind", cc), outstandingWatches(0), latencies(1000), readLatencies(1000),
+    transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), outstandingWatches(0),
-    commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), mvCacheInsertLocation(0),
+    latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0), internal(internal), transactionTracingEnabled(true),
+    bytesPerCommit(1000), mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
-    smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
+    internal(internal), transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
@@ -980,6 +980,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 		                                           [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			                                           if (ryw->getDatabase().getPtr() &&
 			                                               ryw->getDatabase()->getConnectionFile()) {
+				                                           ++ryw->getDatabase()->transactionStatusRequests;
 				                                           return getJSON(ryw->getDatabase());
 			                                           } else {
 				                                           return Optional<Value>();
@@ -1048,13 +1049,14 @@ DatabaseContext::DatabaseContext(const Error& err)
     transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc),
     transactionKeyServerLocationRequests("KeyServerLocationRequests", cc),
     transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc),
-    transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc),
+    transactionStatusRequests("StatusRequests", cc), transactionsTooOld("TooOld", cc),
-    transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc),
+    transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
-    transactionsResourceConstrained("ResourceConstrained", cc), transactionsThrottled("Throttled", cc),
+    transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
-    transactionsProcessBehind("ProcessBehind", cc), latencies(1000), readLatencies(1000), commitLatencies(1000),
+    transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), latencies(1000),
-    GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
+    readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
-    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(false), transactionTracingEnabled(true) {}
+    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(false),
+    transactionTracingEnabled(true) {}
 
 Database DatabaseContext::create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor, LocalityData clientLocality, bool enableLocalityLoadBalance, TaskPriority taskID, bool lockAware, int apiVersion, bool switchable) {
 	return Database( new DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>>(), clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, true, apiVersion, switchable ) );
@@ -1110,7 +1112,7 @@ bool DatabaseContext::getCachedLocations( const KeyRangeRef& range, vector<std::
 Reference<LocationInfo> DatabaseContext::setCachedLocation( const KeyRangeRef& keys, const vector<StorageServerInterface>& servers ) {
 	vector<Reference<ReferencedInterface<StorageServerInterface>>> serverRefs;
 	serverRefs.reserve(servers.size());
-	for(auto& interf : servers) {
+	for (const auto& interf : servers) {
 		serverRefs.push_back( StorageServerInfo::getInterface( this, interf, clientLocality ) );
 	}
 
@@ -1217,11 +1219,11 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional<Str
 				validateOptionValue(value, false);
 				snapshotRywEnabled--;
 				break;
-			case FDBDatabaseOptions::TRANSACTION_TRACE_ENABLE:
+			case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_ENABLE:
 				validateOptionValue(value, false);
 				transactionTracingEnabled++;
 				break;
-			case FDBDatabaseOptions::TRANSACTION_TRACE_DISABLE:
+			case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_DISABLE:
 				validateOptionValue(value, false);
 				transactionTracingEnabled--;
 				break;
@@ -1448,22 +1450,17 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 
 			std::string knobName = optionValue.substr(0, eq);
 			std::string knobValue = optionValue.substr(eq+1);
-			if (const_cast<FlowKnobs*>(FLOW_KNOBS)->setKnob(knobName, knobValue))
+		    if (globalFlowKnobs->setKnob(knobName, knobValue)) {
-			{
+			    // update dependent knobs
-				// update dependent knobs
+			    globalFlowKnobs->initialize();
-				const_cast<FlowKnobs*>(FLOW_KNOBS)->initialize();
+		    } else if (globalClientKnobs->setKnob(knobName, knobValue)) {
-			}
+			    // update dependent knobs
-			else if (const_cast<ClientKnobs*>(CLIENT_KNOBS)->setKnob(knobName, knobValue))
+			    globalClientKnobs->initialize();
-			{
+		    } else {
-				// update dependent knobs
+			    TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
-				const_cast<ClientKnobs*>(CLIENT_KNOBS)->initialize();
-			}
-			else
-			{
-				TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
 				fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str());
-			}
+		    }
-			break;
+		    break;
 		}
 		case FDBNetworkOptions::TLS_PLUGIN:
 			validateOptionValue(value, true);
@@ -1545,6 +1542,23 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 			validateOptionValue(value, false);
 			networkOptions.runLoopProfilingEnabled = true;
 			break;
+		case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: {
+			validateOptionValue(value, true);
+			std::string tracer = value.get().toString();
+			if (tracer == "none" || tracer == "disabled") {
+				openTracer(TracerType::DISABLED);
+			} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
+				openTracer(TracerType::LOG_FILE);
+			} else if (tracer == "network_async") {
+				openTracer(TracerType::NETWORK_ASYNC);
+			} else if (tracer == "network_lossy") {
+				openTracer(TracerType::NETWORK_LOSSY);
+			} else {
+				fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", tracer.c_str());
+				throw invalid_option_value();
+			}
+			break;
+		}
 		default:
 			break;
 	}
@@ -1856,17 +1870,17 @@ Future< vector< pair<KeyRange,Reference<LocationInfo>> > > getKeyRangeLocations(
 	}
 
 	bool foundFailed = false;
-	for(auto& it : locations) {
+	for (const auto& [range, locInfo] : locations) {
 		bool onlyEndpointFailed = false;
-		for(int i = 0; i < it.second->size(); i++) {
+		for (int i = 0; i < locInfo->size(); i++) {
-			if( IFailureMonitor::failureMonitor().onlyEndpointFailed(it.second->get(i, member).getEndpoint()) ) {
+			if (IFailureMonitor::failureMonitor().onlyEndpointFailed(locInfo->get(i, member).getEndpoint())) {
 				onlyEndpointFailed = true;
 				break;
 			}
 		}
 
 		if( onlyEndpointFailed ) {
-			cx->invalidateCache( it.first.begin );
+			cx->invalidateCache(range.begin);
 			foundFailed = true;
 		}
 	}
@@ -1917,6 +1931,7 @@ ACTOR Future<Optional<Value>> getValue( Future<Version> version, Key key, Databa
 {
 	state Version ver = wait( version );
 	state Span span("NAPI:getValue"_loc, info.spanID);
+	span.addTag("key"_sr, key);
 	cx->validateVersion(ver);
 
 	loop {
@@ -2541,7 +2556,6 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Reference<Transa
 				}
 
 				++cx->transactionPhysicalReads;
-				++cx->transactionGetRangeRequests;
 				state GetKeyValuesReply rep;
 				try {
 					if (CLIENT_BUGGIFY) {
@@ -4785,3 +4799,57 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exc
 
 	return (ddCheck && coordinatorCheck);
 }
+
+ACTOR Future<Void> addInterfaceActor( std::map<Key,std::pair<Value,ClientLeaderRegInterface>>* address_interface, Reference<FlowLock> connectLock, KeyValue kv) {
+	wait(connectLock->take());
+	state FlowLock::Releaser releaser(*connectLock);
+	state ClientWorkerInterface workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
+	state ClientLeaderRegInterface leaderInterf(workerInterf.address());
+	choose {
+		when( Optional<LeaderInfo> rep = wait( brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())) ) ) {
+			StringRef ip_port =
+				kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key;
+			(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
+
+			if(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+				Key full_ip_port2 =
+				    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
+				StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls")) ? full_ip_port2.removeSuffix(LiteralStringRef(":tls")) : full_ip_port2;
+				(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
+			}
+		}
+		when( wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT)) ) {} // NOTE : change timeout time here if necessary
+	}
+	return Void();
+}
+
+ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
+	// ignore negative value
+	if (duration < 0) duration = 0;
+	// fetch the addresses of all workers
+	state std::map<Key,std::pair<Value,ClientLeaderRegInterface>> address_interface;
+	if (!cx->getConnectionFile())
+		return 0;
+	Standalone<RangeResultRef> kvs = wait(getWorkerInterfaces(cx->getConnectionFile()));
+	ASSERT(!kvs.more);
+	// Note: reuse this knob from fdbcli, change it if necessary
+	Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
+	std::vector<Future<Void>> addInterfs;
+	for( const auto& it : kvs ) {
+		addInterfs.push_back(addInterfaceActor(&address_interface, connectLock, it));
+	}
+	wait(waitForAll(addInterfs));
+	if (!address_interface.count(addr)) return 0;
+
+	BinaryReader::fromStringRef<ClientWorkerInterface>(address_interface[addr].first, IncludeVersion())
+	    .reboot.send(RebootRequest(false, check, duration));
+	return 1;
+}
+
+Future<int64_t> DatabaseContext::rebootWorker(StringRef addr, bool check, int duration) {
+	return rebootWorkerActor(this, addr, check, duration);
+}
+
+Future<Void> DatabaseContext::forceRecoveryWithDataLoss(StringRef dcId) {
+	return forceRecovery(getConnectionFile(), dcId);
+}

fdbclient/ReadYourWrites.actor.cpp

@@ -1238,6 +1238,7 @@ Future< Optional<Value> > ReadYourWritesTransaction::get( const Key& key, bool s
 	} else {
 		if (key == LiteralStringRef("\xff\xff/status/json")) {
 			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
+				++tr.getDatabase()->transactionStatusRequests;
 				return getJSON(tr.getDatabase());
 			} else {
 				return Optional<Value>();

fdbclient/ReadYourWrites.h

@@ -58,7 +58,9 @@ struct TransactionDebugInfo : public ReferenceCounted<TransactionDebugInfo> {
 
 //Values returned by a ReadYourWritesTransaction will contain a reference to the transaction's arena. Therefore, keeping a reference to a value
 //longer than its creating transaction would hold all of the memory generated by the transaction
-class ReadYourWritesTransaction : NonCopyable, public ReferenceCounted<ReadYourWritesTransaction>, public FastAllocated<ReadYourWritesTransaction> {
+class ReadYourWritesTransaction final : NonCopyable,
+                                        public ReferenceCounted<ReadYourWritesTransaction>,
+                                        public FastAllocated<ReadYourWritesTransaction> {
 public:
 	static ReadYourWritesTransaction* allocateOnForeignThread() {
 		ReadYourWritesTransaction *tr = (ReadYourWritesTransaction*)ReadYourWritesTransaction::operator new( sizeof(ReadYourWritesTransaction) );
@@ -115,9 +117,6 @@ public:
 	void operator=(ReadYourWritesTransaction&& r) noexcept;
 	ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept;
 
-	virtual void addref() { ReferenceCounted<ReadYourWritesTransaction>::addref(); }
-	virtual void delref() { ReferenceCounted<ReadYourWritesTransaction>::delref(); }
-
 	void cancel();
 	void reset();
 	void debugTransaction(UID dID) { tr.debugTransaction(dID); }

fdbclient/S3BlobStore.actor.cpp

@@ -249,7 +249,7 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(std::string const
 	}
 }
 
-std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::string params) {
+std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::string params) const {
 	std::string hostPort = host;
 	if (!service.empty()) {
 		hostPort.append(":");
@@ -271,14 +271,14 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin
 		params.append(knobParams);
 	}
 
-	for (auto& kv : extraHeaders) {
+	for (const auto& [k, v] : extraHeaders) {
 		if (!params.empty()) {
 			params.append("&");
 		}
 		params.append("header=");
-		params.append(HTTP::urlEncode(kv.first));
+		params.append(HTTP::urlEncode(k));
 		params.append(":");
-		params.append(HTTP::urlEncode(kv.second));
+		params.append(HTTP::urlEncode(v));
 	}
 
 	if (!params.empty()) r.append("?").append(params);
@@ -563,12 +563,12 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 	headers["Accept"] = "application/xml";
 
 	// Merge extraHeaders into headers
-	for (auto& kv : bstore->extraHeaders) {
+	for (const auto& [k, v] : bstore->extraHeaders) {
-		std::string& fieldValue = headers[kv.first];
+		std::string& fieldValue = headers[k];
 		if (!fieldValue.empty()) {
 			fieldValue.append(",");
 		}
-		fieldValue.append(kv.second);
+		fieldValue.append(v);
 	}
 
 	// For requests with content to upload, the request timeout should be at least twice the amount of time

fdbclient/S3BlobStore.h

@@ -123,7 +123,7 @@ public:
 
 	// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
 	// parameters in addition to the passed params string
-	std::string getResourceURL(std::string resource, std::string params);
+	std::string getResourceURL(std::string resource, std::string params) const;
 
 	struct ReusableConnection {
 		Reference<IConnection> conn;

fdbclient/SpecialKeySpace.actor.h

@@ -93,14 +93,14 @@ public:
 
 	explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-	virtual ~SpecialKeyRangeRWImpl() {}
+	~SpecialKeyRangeRWImpl() override {}
 };
 
 class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
 public:
 	explicit SpecialKeyRangeAsyncImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-	Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const = 0;
+	Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override = 0;
 
 	// calling with a cache object to have consistent results if we need to call rpc
 	Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr,

fdbclient/StorageServerInterface.h

@@ -451,7 +451,7 @@ struct ReadHotRangeWithMetrics {
 	ReadHotRangeWithMetrics(Arena& arena, const ReadHotRangeWithMetrics& rhs)
 	  : keys(arena, rhs.keys), density(rhs.density), readBandwidth(rhs.readBandwidth) {}
 
-	int expectedSize() { return keys.expectedSize() + sizeof(density) + sizeof(readBandwidth); }
+	int expectedSize() const { return keys.expectedSize() + sizeof(density) + sizeof(readBandwidth); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {

fdbclient/SystemData.cpp

@@ -203,10 +203,6 @@ const KeyRangeRef writeConflictRangeKeysRange =
                 LiteralStringRef("\xff\xff/transaction/write_conflict_range/\xff\xff"));
 
 // "\xff/cacheServer/[[UID]] := StorageServerInterface"
-// This will be added by the cache server on initialization and removed by DD
-// TODO[mpilman]: We will need a way to map uint16_t ids to UIDs in a future
-//                versions. For now caches simply cache everything so the ids
-//                are not yet meaningful.
 const KeyRangeRef storageCacheServerKeys(LiteralStringRef("\xff/cacheServer/"),
                                          LiteralStringRef("\xff/cacheServer0"));
 const KeyRef storageCacheServersPrefix = storageCacheServerKeys.begin;
@@ -598,6 +594,8 @@ ProcessClass decodeProcessClassValue( ValueRef const& value ) {
 const KeyRangeRef configKeys( LiteralStringRef("\xff/conf/"), LiteralStringRef("\xff/conf0") );
 const KeyRef configKeysPrefix = configKeys.begin;
 
+const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint"));
+
 const KeyRangeRef excludedServersKeys( LiteralStringRef("\xff/conf/excluded/"), LiteralStringRef("\xff/conf/excluded0") );
 const KeyRef excludedServersPrefix = excludedServersKeys.begin;
 const KeyRef excludedServersVersionKey = LiteralStringRef("\xff/conf/excluded");

fdbclient/SystemData.h

@@ -45,6 +45,9 @@ extern const KeyRangeRef specialKeys; // [FF][FF] to [FF][FF][FF], some client f
 extern const KeyRef afterAllKeys;
 
 //    "\xff/keyServers/[[begin]]" := "[[vector<serverID>, vector<serverID>]|[vector<Tag>, vector<Tag>]]"
+//	An internal mapping of where shards are located in the database. [[begin]] is the start of the shard range
+//	and the result is a list of serverIDs or Tags where these shards are located. These values can be changed
+//	as data movement occurs.
 extern const KeyRangeRef keyServersKeys, keyServersKeyServersKeys;
 extern const KeyRef keyServersPrefix, keyServersEnd, keyServersKeyServersKey;
 const Key keyServersKey( const KeyRef& k );
@@ -63,6 +66,10 @@ void decodeKeyServersValue( std::map<Tag, UID> const& tag_uid, const ValueRef& v
                             std::vector<UID>& src, std::vector<UID>& dest );
 
 // "\xff/storageCacheServer/[[UID]] := StorageServerInterface"
+// This will be added by the cache server on initialization and removed by DD
+// TODO[mpilman]: We will need a way to map uint16_t ids to UIDs in a future
+//                versions. For now caches simply cache everything so the ids
+//                are not yet meaningful.
 extern const KeyRangeRef storageCacheServerKeys;
 extern const KeyRef storageCacheServersPrefix, storageCacheServersEnd;
 const Key storageCacheServerKey(UID id);
@@ -75,7 +82,11 @@ const Key storageCacheKey( const KeyRef& k );
 const Value storageCacheValue( const std::vector<uint16_t>& serverIndices );
 void decodeStorageCacheValue( const ValueRef& value, std::vector<uint16_t>& serverIndices );
 
-//    "\xff/serverKeys/[[serverID]]/[[begin]]" := "" | "1" | "2"
+//    "\xff/serverKeys/[[serverID]]/[[begin]]" := "[[serverKeysTrue]]" |" [[serverKeysFalse]]"
+//	An internal mapping of what shards any given server currently has ownership of
+//	Using the serverID as a prefix, then followed by the beginning of the shard range
+//	as the key, the value indicates whether the shard does or does not exist on the server.
+//	These values can be changed as data movement occurs.
 extern const KeyRef serverKeysPrefix;
 extern const ValueRef serverKeysTrue, serverKeysFalse;
 const Key serverKeysKey( UID serverID, const KeyRef& keys );
@@ -103,6 +114,8 @@ const Key cacheChangeKeyFor( uint16_t idx );
 uint16_t cacheChangeKeyDecodeIndex( const KeyRef& key );
 
 // "\xff/serverTag/[[serverID]]" = "[[Tag]]"
+//	Provides the Tag for the given serverID. Used to access a
+//	storage server's corresponding TLog in order to apply mutations.
 extern const KeyRangeRef serverTagKeys;
 extern const KeyRef serverTagPrefix;
 extern const KeyRangeRef serverTagMaxKeys;
@@ -122,6 +135,8 @@ Tag decodeServerTagValue( ValueRef const& );
 const Key serverTagConflictKeyFor( Tag );
 
 //    "\xff/tagLocalityList/[[datacenterID]]" := "[[tagLocality]]"
+//	Provides the tagLocality for the given datacenterID
+//	See "FDBTypes.h" struct Tag for more details on tagLocality
 extern const KeyRangeRef tagLocalityListKeys;
 extern const KeyRef tagLocalityListPrefix;
 const Key tagLocalityListKeyFor( Optional<Value> dcID );
@@ -130,6 +145,8 @@ Optional<Value> decodeTagLocalityListKey( KeyRef const& );
 int8_t decodeTagLocalityListValue( ValueRef const& );
 
 //    "\xff\x02/datacenterReplicas/[[datacenterID]]" := "[[replicas]]"
+//	Provides the number of replicas for the given datacenterID.
+//	Used in the initialization of the Data Distributor.
 extern const KeyRangeRef datacenterReplicasKeys;
 extern const KeyRef datacenterReplicasPrefix;
 const Key datacenterReplicasKeyFor( Optional<Value> dcID );
@@ -138,6 +155,8 @@ Optional<Value> decodeDatacenterReplicasKey( KeyRef const& );
 int decodeDatacenterReplicasValue( ValueRef const& );
 
 //    "\xff\x02/tLogDatacenters/[[datacenterID]]"
+//	The existence of an empty string as a value signifies that the datacenterID is valid
+//	(as opposed to having no value at all)
 extern const KeyRangeRef tLogDatacentersKeys;
 extern const KeyRef tLogDatacentersPrefix;
 const Key tLogDatacentersKeyFor( Optional<Value> dcID );
@@ -170,29 +189,46 @@ ProcessClass decodeProcessClassValue( ValueRef const& );
 UID decodeProcessClassKeyOld( KeyRef const& key );
 
 //   "\xff/conf/[[option]]" := "value"
+//	An umbrella prefix for options mostly used by the DatabaseConfiguration class.
+//	See DatabaseConfiguration.cpp ::setInternal for more examples.
 extern const KeyRangeRef configKeys;
 extern const KeyRef configKeysPrefix;
 
+// Change the value of this key to anything and that will trigger detailed data distribution team info log.
+extern const KeyRef triggerDDTeamInfoPrintKey;
+
+//	The differences between excluded and failed can be found in "command-line-interface.rst"
+//	and in the help message of the fdbcli command "exclude".
+
 //   "\xff/conf/excluded/1.2.3.4" := ""
 //   "\xff/conf/excluded/1.2.3.4:4000" := ""
 //   These are inside configKeysPrefix since they represent a form of configuration and they are convenient
 //   to track in the same way by the tlog and recovery process, but they are ignored by the DatabaseConfiguration
 //   class.
+//	 The existence of an empty string as a value signifies that the provided IP has been excluded.
+//	 (as opposed to having no value at all)
 extern const KeyRef excludedServersPrefix;
 extern const KeyRangeRef excludedServersKeys;
 extern const KeyRef excludedServersVersionKey;  // The value of this key shall be changed by any transaction that modifies the excluded servers list
 const AddressExclusion decodeExcludedServersKey( KeyRef const& key ); // where key.startsWith(excludedServersPrefix)
 std::string encodeExcludedServersKey( AddressExclusion const& );
 
+//   "\xff/conf/failed/1.2.3.4" := ""
+//   "\xff/conf/failed/1.2.3.4:4000" := ""
+//   These are inside configKeysPrefix since they represent a form of configuration and they are convenient
+//   to track in the same way by the tlog and recovery process, but they are ignored by the DatabaseConfiguration
+//   class.
+//	 The existence of an empty string as a value signifies that the provided IP has been marked as failed.
+//	 (as opposed to having no value at all)
 extern const KeyRef failedServersPrefix;
 extern const KeyRangeRef failedServersKeys;
 extern const KeyRef failedServersVersionKey;  // The value of this key shall be changed by any transaction that modifies the failed servers list
 const AddressExclusion decodeFailedServersKey( KeyRef const& key ); // where key.startsWith(failedServersPrefix)
 std::string encodeFailedServersKey( AddressExclusion const& );
 
-//    "\xff/workers/[[processID]]" := ""
+//	"\xff/workers/[[processID]]" := ""
-//    Asynchronously updated by the cluster controller, this is a list of fdbserver processes that have joined the cluster
+//	Asynchronously updated by the cluster controller, this is a list of fdbserver processes that have joined the cluster
-//    and are currently (recently) available
+//	and are currently (recently) available
 extern const KeyRangeRef workerListKeys;
 extern const KeyRef workerListPrefix;
 const Key workerListKeyFor(StringRef processID );
@@ -200,7 +236,9 @@ const Value workerListValue( ProcessData const& );
 Key decodeWorkerListKey( KeyRef const& );
 ProcessData decodeWorkerListValue( ValueRef const& );
 
-//    "\xff\x02/backupProgress/[[workerID]]" := "[[WorkerBackupStatus]]"
+//	"\xff\x02/backupProgress/[[workerID]]" := "[[WorkerBackupStatus]]"
+//	Provides the progress for the given backup worker.
+//	See "FDBTypes.h" struct WorkerBackupStatus for more details on the return type value.
 extern const KeyRangeRef backupProgressKeys;
 extern const KeyRef backupProgressPrefix;
 const Key backupProgressKeyFor(UID workerID);
@@ -214,18 +252,31 @@ extern const KeyRef backupStartedKey;
 Value encodeBackupStartedValue(const std::vector<std::pair<UID, Version>>& ids);
 std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& value);
 
-// The key to signal backup workers that they should pause or resume.
+// The key to signal backup workers that they should resume or pause.
 //    "\xff\x02/backupPaused" := "[[0|1]]"
+// 0 = Send a signal to resume/already resumed.
+// 1 = Send a signal to pause/already paused.
 extern const KeyRef backupPausedKey;
 
+//	"\xff/coordinators" = "[[ClusterConnectionString]]"
+//	Set to the encoded structure of the cluster's current set of coordinators.
+//	Changed when performing quorumChange.
+//	See "CoordinationInterface.h" struct ClusterConnectionString for more details
 extern const KeyRef coordinatorsKey;
+
+//	"\xff/logs" = "[[LogsValue]]"
+//	Used during master recovery in order to communicate
+//	and store info about the logs system.
 extern const KeyRef logsKey;
+
+//	"\xff/minRequiredCommitVersion" = "[[Version]]"
+//	Used during backup/recovery to restrict version requirements
 extern const KeyRef minRequiredCommitVersionKey;
 
 const Value logsValue( const vector<std::pair<UID, NetworkAddress>>& logs, const vector<std::pair<UID, NetworkAddress>>& oldLogs );
 std::pair<vector<std::pair<UID, NetworkAddress>>,vector<std::pair<UID, NetworkAddress>>> decodeLogsValue( const ValueRef& value );
 
-// The "global keys" are send to each storage server any time they are changed
+// The "global keys" are sent to each storage server any time they are changed
 extern const KeyRef globalKeysPrefix;
 extern const KeyRef lastEpochEndKey;
 extern const KeyRef lastEpochEndPrivateKey;
@@ -253,6 +304,7 @@ extern const KeyRef tagThrottleLimitKey;
 extern const KeyRef tagThrottleCountKey;
 
 // Log Range constant variables
+// Used in the backup pipeline to track mutations
 // \xff/logRanges/[16-byte UID][begin key] := serialize( make_pair([end key], [destination key prefix]), IncludeVersion() )
 extern const KeyRangeRef logRangesRange;
 
@@ -397,8 +449,16 @@ std::pair<Key,Version> decodeHealthyZoneValue( ValueRef const& );
 extern const KeyRangeRef testOnlyTxnStateStorePrefixRange;
 
 // Snapshot + Incremental Restore
+
+//	"\xff/writeRecovery" = "[[writeRecoveryKeyTrue]]"
+//	Flag used for the snapshot-restore pipeline in order to avoid
+//	anomalous behaviour with multiple recoveries.
 extern const KeyRef writeRecoveryKey;
 extern const ValueRef writeRecoveryKeyTrue;
+
+//	"\xff/snapshotEndVersion" = "[[Version]]"
+//	Written by master server during recovery if recovering from a snapshot.
+//	Allows incremental restore to read and set starting version for consistency. 
 extern const KeyRef snapshotEndVersionKey;
 
 #pragma clang diagnostic pop

fdbclient/TagThrottle.h

@@ -169,7 +169,7 @@ struct TagThrottleValue {
 	template<class Ar>
 	void serialize(Ar& ar) {
 		if(ar.protocolVersion().hasTagThrottleValueReason()) {
-			serializer(ar, tpsRate, expirationTime, initialDuration, reinterpret_cast<uint8_t&>(reason));
+			serializer(ar, tpsRate, expirationTime, initialDuration, reason);
 		}
 		else if(ar.protocolVersion().hasTagThrottleValue()) {
 			serializer(ar, tpsRate, expirationTime, initialDuration);
@@ -216,8 +216,6 @@ namespace ThrottleApi {
 	Future<Void> enableAuto(Database const& db, bool const& enabled);
 };
 
-BINARY_SERIALIZABLE(TransactionPriority);
-
 template<class Value>
 using TransactionTagMap = std::unordered_map<TransactionTag, Value, std::hash<TransactionTagRef>>;
 

fdbclient/TaskBucket.actor.cpp

@@ -29,9 +29,15 @@ Reference<TaskFuture> Task::getDoneFuture(Reference<FutureBucket> fb) {
 struct UnblockFutureTaskFunc : TaskFuncBase {
 	static StringRef name;
 
-	StringRef getName() const { return name; };
+	StringRef getName() const override { return name; };
-	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
-	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+	                     Reference<Task> task) override {
+		return Void();
+	};
+	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
+	                    Reference<Task> task) override {
+		return _finish(tr, tb, fb, task);
+	};
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 		state Reference<TaskFuture> future = futureBucket->unpack(task->params[Task::reservedTaskParamKeyFuture]);
@@ -55,9 +61,13 @@ REGISTER_TASKFUNC(UnblockFutureTaskFunc);
 struct AddTaskFunc : TaskFuncBase {
 	static StringRef name;
 
-	StringRef getName() const { return name; };
+	StringRef getName() const override { return name; };
-	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
-	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { 
+	                     Reference<Task> task) override {
+		return Void();
+	};
+	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
+	                    Reference<Task> task) override {
 		task->params[Task::reservedTaskParamKeyType] = task->params[Task::reservedTaskParamKeyAddTask];
 		tb->addTask(tr, task);
 		return Void();
@@ -71,9 +81,15 @@ struct IdleTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	StringRef getName() const { return name; };
+	StringRef getName() const override { return name; };
-	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
+	Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
-	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return tb->finish(tr, task); };
+	                     Reference<Task> task) override {
+		return Void();
+	};
+	Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
+	                    Reference<Task> task) override {
+		return tb->finish(tr, task);
+	};
 };
 StringRef IdleTaskFunc::name = LiteralStringRef("idle");
 REGISTER_TASKFUNC(IdleTaskFunc);

fdbclient/ThreadSafeTransaction.cpp

@@ -68,6 +68,22 @@ void ThreadSafeDatabase::setOption( FDBDatabaseOptions::Option option, Optional<
 	}, &db->deferredError );
 }
 
+ThreadFuture<int64_t> ThreadSafeDatabase::rebootWorker(const StringRef& address, bool check, int duration) {
+	DatabaseContext *db = this->db;
+	Key addressKey = address;
+	return onMainThread( [db, addressKey, check, duration]() -> Future<int64_t> {
+		return db->rebootWorker(addressKey, check, duration);
+	} );
+}
+
+ThreadFuture<Void> ThreadSafeDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
+	DatabaseContext *db = this->db;
+	Key dcidKey = dcid;
+	return onMainThread( [db, dcidKey]() -> Future<Void> {
+		return db->forceRecoveryWithDataLoss(dcidKey);
+	} );
+}
+
 ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) {
 	ClusterConnectionFile *connFile = new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first);
 

fdbclient/ThreadSafeTransaction.h

@@ -29,17 +29,20 @@
 
 class ThreadSafeDatabase : public IDatabase, public ThreadSafeReferenceCounted<ThreadSafeDatabase> {
 public:
-	~ThreadSafeDatabase();
+	~ThreadSafeDatabase() override;
 	static ThreadFuture<Reference<IDatabase>> createFromExistingDatabase(Database cx);
 
-	Reference<ITransaction> createTransaction();
+	Reference<ITransaction> createTransaction() override;
 
-	void setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>() );
+	void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
 
 	ThreadFuture<Void> onConnected();  // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable.
 
-	void addref() { ThreadSafeReferenceCounted<ThreadSafeDatabase>::addref(); }
+	void addref() override { ThreadSafeReferenceCounted<ThreadSafeDatabase>::addref(); }
-	void delref() { ThreadSafeReferenceCounted<ThreadSafeDatabase>::delref(); }
+	void delref() override { ThreadSafeReferenceCounted<ThreadSafeDatabase>::delref(); }
+
+	ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
+	ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
 
 private:
 	friend class ThreadSafeTransaction;
@@ -53,7 +56,7 @@ public:  // Internal use only
 class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted<ThreadSafeTransaction>, NonCopyable {
 public:
 	explicit ThreadSafeTransaction(DatabaseContext* cx);
-	~ThreadSafeTransaction();
+	~ThreadSafeTransaction() override;
 
 	void cancel() override;
 	void setVersion( Version v ) override;
@@ -115,18 +118,18 @@ private:
 
 class ThreadSafeApi : public IClientApi, ThreadSafeReferenceCounted<ThreadSafeApi> {
 public:
-	void selectApiVersion(int apiVersion);
+	void selectApiVersion(int apiVersion) override;
-	const char* getClientVersion();
+	const char* getClientVersion() override;
 	ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) override;
 
-	void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
+	void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
-	void setupNetwork();
+	void setupNetwork() override;
-	void runNetwork();
+	void runNetwork() override;
-	void stopNetwork();
+	void stopNetwork() override;
 
-	Reference<IDatabase> createDatabase(const char *clusterFilePath);
+	Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
 
-	void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter);
+	void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
 
 	static IClientApi* api;
 

fdbclient/vexillographer/c.cs

@@ -52,8 +52,8 @@ namespace vexillographer
         {
             string parameterComment = "";
             if (o.scope.ToString().EndsWith("Option"))
-                parameterComment = String.Format("{0}// {1}\n", indent, "Parameter: " + o.getParameterComment());
+                parameterComment = String.Format("{0}/* {1} */\n", indent, "Parameter: " + o.getParameterComment());
-            return String.Format("{0}// {2}\n{5}{0}{1}{3}={4}", indent, prefix, o.comment, o.name.ToUpper(), o.code, parameterComment);
+            return String.Format("{0}/* {2} */\n{5}{0}{1}{3}={4}", indent, prefix, o.comment, o.name.ToUpper(), o.code, parameterComment);
         }
 
         private static void writeCEnum(TextWriter outFile, Scope scope, IEnumerable<Option> options)

fdbclient/vexillographer/fdb.options

@@ -125,6 +125,9 @@ description is not currently required but encouraged.
     <Option name="client_buggify_section_fired_probability" code="83"
             paramType="Int" paramDescription="probability expressed as a percentage between 0 and 100"
             description="Set the probability of an active CLIENT_BUGGIFY section being fired. A section will only fire if it was activated" />
+    <Option name="distributed_client_tracer" code="90"
+            paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, network_async, or network_lossy"
+            description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
     <Option name="supported_client_versions" code="1000"
             paramType="String" paramDescription="[release version],[source version],[protocol version];..."
             description="This option is set automatically to communicate the list of supported clients to the active client."
@@ -182,9 +185,9 @@ description is not currently required but encouraged.
     <Option name="transaction_include_port_in_address" code="505"
             description="Addresses returned by get_addresses_for_key include the port when enabled. As of api version 630, this option is enabled by default and setting this has no effect."
             defaultFor="23"/>
-    <Option name="transaction_trace_enable" code="600"
+    <Option name="distributed_transaction_trace_enable" code="600"
             description="Enable tracing for all transactions. This is the default." />
-    <Option name="transaction_trace_disable" code="601"
+    <Option name="distributed_transaction_trace_disable" code="601"
             description="Disable tracing for all transactions." />
   </Scope>
   

fdbrpc/AsyncFileCached.actor.h

@@ -142,7 +142,7 @@ struct OpenFileInfo : NonCopyable {
 
 struct AFCPage;
 
-class AsyncFileCached : public IAsyncFile, public ReferenceCounted<AsyncFileCached> {
+class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFileCached> {
 	friend struct AFCPage;
 
 public:
@@ -221,11 +221,11 @@ public:
 
 	std::string getFilename() const override { return filename; }
 
-	virtual void addref() { 
+	void addref() override {
 		ReferenceCounted<AsyncFileCached>::addref(); 
 		//TraceEvent("AsyncFileCachedAddRef").detail("Filename", filename).detail("Refcount", debugGetReferenceCount()).backtrace();
 	}
-	virtual void delref() {
+	void delref() override {
 		if (delref_no_destroy()) {
 			// If this is ever ThreadSafeReferenceCounted...
 			// setrefCountUnsafe(0);
@@ -240,7 +240,7 @@ public:
 		}
 	}
 
-	~AsyncFileCached();
+	~AsyncFileCached() override;
 
 private:
 	static std::map< std::string, OpenFileInfo > openFiles;
@@ -570,7 +570,7 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		pageCache->allocate(this);
 	}
 
-	virtual ~AFCPage() {
+	~AFCPage() override {
 		clearDirty();
 		ASSERT_ABORT( flushableIndex == -1 );
 	}

fdbrpc/AsyncFileEIO.actor.h

@@ -199,7 +199,7 @@ public:
 		return dispatch_impl(func);
 	}
 
-	~AsyncFileEIO() { close_impl( fd ); }
+	~AsyncFileEIO() override { close_impl(fd); }
 
 private:
 	struct ErrorInfo : ReferenceCounted<ErrorInfo>, FastAllocated<ErrorInfo> {

fdbrpc/AsyncFileKAIO.actor.h

@@ -52,7 +52,7 @@ DESCR struct SlowAioSubmit {
 	int64_t largestTruncate;
 };
 
-class AsyncFileKAIO : public IAsyncFile, public ReferenceCounted<AsyncFileKAIO> {
+class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFileKAIO> {
 public:
 
 #if KAIO_LOGGING
@@ -179,8 +179,8 @@ public:
 	static int get_eventfd() { return ctx.evfd; }
 	static void setTimeout(double ioTimeout) { ctx.setIOTimeout(ioTimeout); }
 
-	virtual void addref() { ReferenceCounted<AsyncFileKAIO>::addref(); }
+	void addref() override { ReferenceCounted<AsyncFileKAIO>::addref(); }
-	virtual void delref() { ReferenceCounted<AsyncFileKAIO>::delref(); }
+	void delref() override { ReferenceCounted<AsyncFileKAIO>::delref(); }
 
 	Future<int> read(void* data, int length, int64_t offset) override {
 		++countFileLogicalReads;
@@ -343,7 +343,7 @@ public:
 	Future<int64_t> size() const override { return nextFileSize; }
 	int64_t debugFD() const override { return fd; }
 	std::string getFilename() const override { return filename; }
-	~AsyncFileKAIO() {
+	~AsyncFileKAIO() override {
 		close(fd);
 
 #if KAIO_LOGGING

fdbrpc/AsyncFileNonDurable.actor.h

@@ -53,7 +53,7 @@ Future<T> sendErrorOnShutdown( Future<T> in ) {
 	}
 }
 
-class AsyncFileDetachable sealed : public IAsyncFile, public ReferenceCounted<AsyncFileDetachable>{
+class AsyncFileDetachable final : public IAsyncFile, public ReferenceCounted<AsyncFileDetachable> {
 private:
 	Reference<IAsyncFile> file;
 	Future<Void> shutdown;
@@ -125,7 +125,7 @@ public:
 
 //An async file implementation which wraps another async file and will randomly destroy sectors that it is writing when killed
 //This is used to simulate a power failure which prevents all written data from being persisted to disk
-class AsyncFileNonDurable sealed : public IAsyncFile, public ReferenceCounted<AsyncFileNonDurable>{
+class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<AsyncFileNonDurable> {
 public:
 	UID id;
 	std::string filename;
@@ -239,7 +239,7 @@ public:
 		}
 	}
 
-	~AsyncFileNonDurable() {
+	~AsyncFileNonDurable() override {
 		//TraceEvent("AsyncFileNonDurable_Destroy", id).detail("Filename", filename);
 	}
 
@@ -288,7 +288,7 @@ public:
 
 	//Fsyncs the file.  This allows all delayed modifications to the file to complete before
 	//syncing the underlying file
-	Future<Void> sync() {
+	Future<Void> sync() override {
 		//TraceEvent("AsyncFileNonDurable_Sync", id).detail("Filename", filename);
 		Future<Void> syncFuture = sync(this, true);
 		reponses.add( syncFuture );

fdbrpc/AsyncFileReadAhead.actor.h

@@ -177,7 +177,7 @@ public:
 
 	std::string getFilename() const override { return m_f->getFilename(); }
 
-	~AsyncFileReadAheadCache() {
+	~AsyncFileReadAheadCache() override {
 		for(auto &it : m_blocks) {
 			it.second.cancel();
 		}

fdbrpc/AsyncFileWinASIO.actor.h

@@ -35,7 +35,7 @@
 #undef min
 #undef max
 
-class AsyncFileWinASIO : public IAsyncFile, public ReferenceCounted<AsyncFileWinASIO> {
+class AsyncFileWinASIO final : public IAsyncFile, public ReferenceCounted<AsyncFileWinASIO> {
 public:
 	static void init() {}
 
@@ -84,8 +84,8 @@ public:
 		return buf.st_mtime;
 	}
 
-	virtual void addref() { ReferenceCounted<AsyncFileWinASIO>::addref(); }
+	void addref() override { ReferenceCounted<AsyncFileWinASIO>::addref(); }
-	virtual void delref() { ReferenceCounted<AsyncFileWinASIO>::delref(); }
+	void delref() override { ReferenceCounted<AsyncFileWinASIO>::delref(); }
 
 	int64_t debugFD() const override { return (int64_t)(const_cast<decltype(file)&>(file).native_handle()); }
 

fdbrpc/AsyncFileWriteChecker.h

@@ -27,26 +27,26 @@
 
 class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFileWriteChecker> {
 public:
-	void addref() { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
+	void addref() override { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
-	void delref() { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
+	void delref() override { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
 
 	// For read() and write(), the data buffer must remain valid until the future is ready
-	Future<int> read( void* data, int length, int64_t offset ) {
+	Future<int> read(void* data, int length, int64_t offset) override {
 		return map(m_f->read(data, length, offset), [=](int r) {
 			updateChecksumHistory(false, offset, r, (uint8_t*)data);
 			return r;
 		});
 	}
-	Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
+	Future<Void> readZeroCopy(void** data, int* length, int64_t offset) override {
 		return map(m_f->readZeroCopy(data, length, offset), [=](Void r) { updateChecksumHistory(false, offset, *length, (uint8_t *)data); return r; });
 	}
 
-	Future<Void> write( void const* data, int length, int64_t offset ) {
+	Future<Void> write(void const* data, int length, int64_t offset) override {
 		updateChecksumHistory(true, offset, length, (uint8_t *)data);
 		return m_f->write(data, length, offset);
 	}
 
-	Future<Void> truncate( int64_t size )  {
+	Future<Void> truncate(int64_t size) override {
 		return map(m_f->truncate(size), [=](Void r) {
 			// Truncate the page checksum history if it is in use
 			if( (size / checksumHistoryPageSize) < checksumHistory.size() ) {
@@ -58,11 +58,13 @@ public:
 		});
 	}
 
-	Future<Void> sync() { return m_f->sync(); }
+	Future<Void> sync() override { return m_f->sync(); }
-	Future<Void> flush() { return m_f->flush(); }
+	Future<Void> flush() override { return m_f->flush(); }
 	Future<int64_t> size() const override { return m_f->size(); }
 	std::string getFilename() const override { return m_f->getFilename(); }
-	void releaseZeroCopy( void* data, int length, int64_t offset )  { return m_f->releaseZeroCopy(data, length, offset); }
+	void releaseZeroCopy(void* data, int length, int64_t offset) override {
+		return m_f->releaseZeroCopy(data, length, offset);
+	}
 	int64_t debugFD() const override { return m_f->debugFD(); }
 
 	AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f) {
@@ -75,7 +77,7 @@ public:
 		checksumHistoryBudget.get() -= checksumHistory.capacity();
 	}
 
-	virtual ~AsyncFileWriteChecker() { checksumHistoryBudget.get() += checksumHistory.capacity(); }
+	~AsyncFileWriteChecker() override { checksumHistoryBudget.get() += checksumHistory.capacity(); }
 
 private:
 	Reference<IAsyncFile> m_f;

fdbrpc/CMakeLists.txt

@@ -22,6 +22,8 @@ set(FDBRPC_SRCS
   ReplicationPolicy.cpp
   ReplicationTypes.cpp
   ReplicationUtils.cpp
+  SimExternalConnection.actor.cpp
+  SimExternalConnection.h
   Stats.actor.cpp
   Stats.h
   sim2.actor.cpp

fdbrpc/FailureMonitor.h

@@ -138,8 +138,8 @@ public:
 class SimpleFailureMonitor : public IFailureMonitor {
 public:
 	SimpleFailureMonitor();
-	void setStatus(NetworkAddress const& address, FailureStatus const& status);
+	void setStatus(NetworkAddress const& address, FailureStatus const& status) override;
-	void endpointNotFound(Endpoint const&);
+	void endpointNotFound(Endpoint const&) override;
 	void notifyDisconnect(NetworkAddress const&) override;
 
 	Future<Void> onStateChanged(Endpoint const& endpoint) override;

fdbrpc/FlowTests.actor.cpp

@@ -85,7 +85,7 @@ class LambdaCallback final : public CallbackType, public FastAllocated<LambdaCal
 		func(t);
 		delete this;
 	}
-	void fire(T&& t) {
+	void fire(T&& t) override {
 		CallbackType::remove();
 		func(std::move(t));
 		delete this;

fdbrpc/FlowTransport.actor.cpp

@@ -855,6 +855,9 @@ static bool checkCompatible(const PeerCompatibilityPolicy& policy, ProtocolVersi
 		return version.version() == policy.version.version();
 	case RequirePeer::AtLeast:
 		return version.version() >= policy.version.version();
+	default:
+		ASSERT(false);
+		return false;
 	}
 }
 

fdbrpc/IAsyncFile.actor.cpp

@@ -108,7 +108,7 @@ ACTOR static Future<Void> incrementalDeleteHelper( std::string filename, bool mu
 	return Void();
 }
 
-Future<Void> IAsyncFileSystem::incrementalDeleteFile( std::string filename, bool mustBeDurable ) {
+Future<Void> IAsyncFileSystem::incrementalDeleteFile(const std::string& filename, bool mustBeDurable) {
 	return uncancellable(incrementalDeleteHelper(
 		filename,
 		mustBeDurable,

fdbrpc/IAsyncFile.h

@@ -88,17 +88,17 @@ typedef void (*runCycleFuncPtr)();
 class IAsyncFileSystem {
 public:
 	// Opens a file for asynchronous I/O
-	virtual Future< Reference<class IAsyncFile> > open( std::string filename, int64_t flags, int64_t mode ) = 0;
+	virtual Future<Reference<class IAsyncFile>> open(const std::string& filename, int64_t flags, int64_t mode) = 0;
 
 	// Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-	virtual Future< Void > deleteFile( std::string filename, bool mustBeDurable ) = 0;
+	virtual Future<Void> deleteFile(const std::string& filename, bool mustBeDurable) = 0;
 
 	// Unlinks a file and then deletes it slowly by truncating the file repeatedly.
 	// If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-	virtual Future<Void> incrementalDeleteFile( std::string filename, bool mustBeDurable );
+	virtual Future<Void> incrementalDeleteFile(const std::string& filename, bool mustBeDurable);
 
 	// Returns the time of the last modification of the file.
-	virtual Future<std::time_t> lastWriteTime( std::string filename ) = 0;
+	virtual Future<std::time_t> lastWriteTime(const std::string& filename) = 0;
 
 	static IAsyncFileSystem* filesystem() { return filesystem(g_network); }
 	static runCycleFuncPtr runCycleFunc() { return reinterpret_cast<runCycleFuncPtr>(reinterpret_cast<flowGlobalType>(g_network->global(INetwork::enRunCycleFunc))); }

fdbrpc/IRateControl.h

@@ -41,7 +41,7 @@ public:
 		m_budget_max = m_limit * m_seconds;
 		m_last_update = timer();
 	}
-	~SpeedLimit() = default;
+	~SpeedLimit() override = default;
 
 	void addref() override { ReferenceCounted<SpeedLimit>::addref(); }
 	void delref() override { ReferenceCounted<SpeedLimit>::delref(); }
@@ -78,7 +78,7 @@ private:
 class Unlimited final : public IRateControl, ReferenceCounted<Unlimited> {
 public:
 	Unlimited() {}
-	~Unlimited() = default;
+	~Unlimited() override = default;
 	void addref() override { ReferenceCounted<Unlimited>::addref(); }
 	void delref() override { ReferenceCounted<Unlimited>::delref(); }
 

fdbrpc/Net2FileSystem.cpp

@@ -40,8 +40,7 @@
 #include "fdbrpc/AsyncFileWriteChecker.h"
 
 // Opens a file for asynchronous I/O
-Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename, int64_t flags, int64_t mode )
+Future<Reference<class IAsyncFile>> Net2FileSystem::open(const std::string& filename, int64_t flags, int64_t mode) {
-{
 #ifdef __linux__
 	if (checkFileSystem) {
 		dev_t fileDeviceId = getDeviceId(filename);
@@ -75,22 +74,19 @@ Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename
 }
 
 // Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-Future< Void > Net2FileSystem::deleteFile( std::string filename, bool mustBeDurable )
+Future<Void> Net2FileSystem::deleteFile(const std::string& filename, bool mustBeDurable) {
-{
 	return Net2AsyncFile::deleteFile(filename, mustBeDurable);
 }
 
-Future< std::time_t > Net2FileSystem::lastWriteTime( std::string filename ) {
+Future<std::time_t> Net2FileSystem::lastWriteTime(const std::string& filename) {
 	return Net2AsyncFile::lastWriteTime( filename );
 }
 
-void Net2FileSystem::newFileSystem(double ioTimeout, std::string fileSystemPath)
+void Net2FileSystem::newFileSystem(double ioTimeout, const std::string& fileSystemPath) {
-{
 	g_network->setGlobal(INetwork::enFileSystem, (flowGlobalType) new Net2FileSystem(ioTimeout, fileSystemPath));
 }
 
-Net2FileSystem::Net2FileSystem(double ioTimeout, std::string fileSystemPath)
+Net2FileSystem::Net2FileSystem(double ioTimeout, const std::string& fileSystemPath) {
-{
 	Net2AsyncFile::init();
 #ifdef __linux__
 	if (!FLOW_KNOBS->DISABLE_POSIX_KERNEL_AIO)

fdbrpc/Net2FileSystem.h

@@ -24,25 +24,25 @@
 
 #include "fdbrpc/IAsyncFile.h"
 
-class Net2FileSystem : public IAsyncFileSystem {
+class Net2FileSystem final : public IAsyncFileSystem {
 public:
 	// Opens a file for asynchronous I/O
-	virtual Future< Reference<class IAsyncFile> > open( std::string filename, int64_t flags, int64_t mode );
+	Future<Reference<class IAsyncFile>> open(const std::string& filename, int64_t flags, int64_t mode) override;
 
 	// Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-	virtual Future< Void > deleteFile( std::string filename, bool mustBeDurable );
+	Future<Void> deleteFile(const std::string& filename, bool mustBeDurable) override;
 
 	// Returns the time of the last modification of the file.
-	virtual Future< std::time_t > lastWriteTime( std::string filename );
+	Future<std::time_t> lastWriteTime(const std::string& filename) override;
 
 	//void init();
 	static void stop();
 
-	Net2FileSystem(double ioTimeout=0.0, std::string fileSystemPath = "");
+	Net2FileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");
 
-	virtual ~Net2FileSystem() {}
+	~Net2FileSystem() override {}
 
-	static void newFileSystem(double ioTimeout=0.0, std::string fileSystemPath = "");
+	static void newFileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");
 
 #ifdef __linux__
 	dev_t fileSystemDeviceId;

fdbrpc/Replication.h

@@ -505,7 +505,7 @@ protected:
 struct LocalityGroup : public LocalitySet {
 	LocalityGroup():LocalitySet(*this), _valuemap(new StringToIntMap()) {}
 	LocalityGroup(LocalityGroup const& source):LocalitySet(source), _recordArray(source._recordArray), _valuemap(source._valuemap) {}
-	virtual ~LocalityGroup() { }
+	~LocalityGroup() override {}
 
 	LocalityEntry const& add(LocalityData const& data) {
 		// _recordArray.size() is the new entry index for the new data
@@ -514,7 +514,7 @@ struct LocalityGroup : public LocalitySet {
 		return LocalitySet::add(record, *this);
 	}
 
-	virtual void clear() {
+	void clear() override {
 		LocalitySet::clear();
 		_valuemap->clear();
 		_recordArray.clear();
@@ -534,15 +534,15 @@ struct LocalityGroup : public LocalitySet {
 		return *this;
 	}
 
-	virtual Reference<LocalityRecord> const& getRecord(int recordIndex) const {
+	Reference<LocalityRecord> const& getRecord(int recordIndex) const override {
 		ASSERT((recordIndex >= 0) && (recordIndex < _recordArray.size()));
 		return _recordArray[recordIndex];
 	}
 
 	// Get the locality info for debug purpose
-	virtual std::vector<Reference<LocalityRecord>> const& getRecordArray() const { return _recordArray; }
+	std::vector<Reference<LocalityRecord>> const& getRecordArray() const override { return _recordArray; }
 
-	virtual int	getMemoryUsed() const {
+	int getMemoryUsed() const override {
 		int memorySize = sizeof(_recordArray) + _keymap->getMemoryUsed();
 		for (auto& record : _recordArray) {
 			memorySize += record->getMemoryUsed();
@@ -563,18 +563,14 @@ struct LocalityGroup : public LocalitySet {
 		return attribHashMap;
 	}
 
-	virtual Reference<StringToIntMap> const&	getGroupValueMap() const
+	Reference<StringToIntMap> const& getGroupValueMap() const override { return _valuemap; }
-	{	return _valuemap; }
 
-	virtual Reference<StringToIntMap> const&	getGroupKeyMap() const
+	Reference<StringToIntMap> const& getGroupKeyMap() const override { return _keymap; }
-	{	return _keymap; }
 
 protected:
-	virtual Reference<StringToIntMap> &	getGroupValueMap()
+	Reference<StringToIntMap>& getGroupValueMap() override { return _valuemap; }
-	{	return _valuemap; }
 
-	virtual Reference<StringToIntMap> &	getGroupKeyMap()
+	Reference<StringToIntMap>& getGroupKeyMap() override { return _keymap; }
-	{	return _keymap; }
 
 protected:
 	std::vector<Reference<LocalityRecord>>	_recordArray;
@@ -585,7 +581,7 @@ template <class V>
 struct LocalityMap : public LocalityGroup  {
 	LocalityMap():LocalityGroup() {}
 	LocalityMap(LocalityMap const& source):LocalityGroup(source), _objectArray(source._objectArray) {}
-	virtual ~LocalityMap() {}
+	~LocalityMap() override {}
 
 	bool selectReplicas(
 		Reference<IReplicationPolicy> const&								policy,
@@ -649,12 +645,12 @@ struct LocalityMap : public LocalityGroup  {
 		return getObject(record->_entryIndex);
 	}
 
-	virtual void clear() {
+	void clear() override {
 		LocalityGroup::clear();
 		_objectArray.clear();
 	}
 
-	virtual int	getMemoryUsed() const {
+	int getMemoryUsed() const override {
 		return LocalitySet::getMemoryUsed() + sizeof(_objectArray) + (sizeof(V*) * _objectArray.size());
 	}
 

fdbrpc/ReplicationPolicy.h

@@ -116,7 +116,7 @@ struct PolicyAcross final : IReplicationPolicy, public ReferenceCounted<PolicyAc
 	PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy);
 	explicit PolicyAcross();
 	explicit PolicyAcross(const PolicyAcross& other) : PolicyAcross(other._count, other._attribKey, other._policy) {}
-	~PolicyAcross();
+	~PolicyAcross() override;
 	std::string name() const override { return "Across"; }
 	std::string embeddedPolicyName() const { return _policy->name(); }
 	int getCount() const { return _count; }

fdbrpc/SimExternalConnection.actor.cpp

@@ -0,0 +1,218 @@
+/*
+ * SimExternalConnection.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define BOOST_SYSTEM_NO_LIB
+#define BOOST_DATE_TIME_NO_LIB
+#define BOOST_REGEX_NO_LIB
+#include <boost/asio.hpp>
+#include <boost/range.hpp>
+#include <thread>
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/SimExternalConnection.h"
+#include "flow/Net2Packet.h"
+#include "flow/Platform.h"
+#include "flow/SendBufferIterator.h"
+#include "flow/UnitTest.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+using namespace boost::asio;
+
+static io_service ios;
+
+class SimExternalConnectionImpl {
+public:
+	ACTOR static Future<Void> onReadable(SimExternalConnection* self) {
+		wait(delayJittered(0.1));
+		if (self->readBuffer.empty()) {
+			wait(self->onReadableTrigger.onTrigger());
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Reference<IConnection>> connect(NetworkAddress toAddr) {
+		wait(delayJittered(0.1));
+		ip::tcp::socket socket(ios);
+		auto ip = toAddr.ip;
+		ip::address address;
+		if (ip.isV6()) {
+			address = boost::asio::ip::address_v6(ip.toV6());
+		} else {
+			address = boost::asio::ip::address_v4(ip.toV4());
+		}
+		boost::system::error_code err;
+		socket.connect(ip::tcp::endpoint(address, toAddr.port), err);
+		if (err) {
+			return Reference<IConnection>();
+		} else {
+			return Reference<IConnection>(new SimExternalConnection(std::move(socket)));
+		}
+	}
+};
+
+void SimExternalConnection::close() {
+	socket.close();
+}
+
+Future<Void> SimExternalConnection::acceptHandshake() {
+	return Void();
+}
+
+Future<Void> SimExternalConnection::connectHandshake() {
+	return Void();
+}
+
+Future<Void> SimExternalConnection::onWritable() {
+	return Void();
+}
+
+Future<Void> SimExternalConnection::onReadable() {
+	return SimExternalConnectionImpl::onReadable(this);
+}
+
+int SimExternalConnection::read(uint8_t* begin, uint8_t* end) {
+	auto toRead = std::min<int>(end - begin, readBuffer.size());
+	// TODO: Improve performance
+	for (int i = 0; i < toRead; ++i) {
+		*(begin + i) = readBuffer.front();
+		readBuffer.pop_front();
+	}
+	return toRead;
+}
+
+int SimExternalConnection::write(SendBuffer const* buffer, int limit) {
+	boost::system::error_code err;
+	bool triggerReaders = (socket.available() == 0);
+	int bytesSent = socket.write_some(
+	    boost::iterator_range<SendBufferIterator>(SendBufferIterator(buffer, limit), SendBufferIterator()), err);
+	ASSERT(!err);
+	ASSERT(bytesSent > 0);
+	threadSleep(0.1);
+	const auto bytesReadable = socket.available();
+	std::vector<uint8_t> tempReadBuffer(bytesReadable);
+	for (int index = 0; index < bytesReadable;) {
+		index += socket.read_some(mutable_buffers_1(&tempReadBuffer[index], bytesReadable), err);
+	}
+	std::copy(tempReadBuffer.begin(), tempReadBuffer.end(), std::inserter(readBuffer, readBuffer.end()));
+	ASSERT(!err);
+	ASSERT(socket.available() == 0);
+	if (triggerReaders) {
+		onReadableTrigger.trigger();
+	}
+	return bytesSent;
+}
+
+NetworkAddress SimExternalConnection::getPeerAddress() const {
+	auto endpoint = socket.remote_endpoint();
+	auto addr = endpoint.address();
+	if (addr.is_v6()) {
+		return NetworkAddress(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
+	} else {
+		return NetworkAddress(addr.to_v4().to_ulong(), endpoint.port());
+	}
+}
+
+UID SimExternalConnection::getDebugID() const {
+	return dbgid;
+}
+
+ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host, std::string service) {
+	wait(delayJittered(0.1));
+	ip::tcp::resolver resolver(ios);
+	ip::tcp::resolver::query query(host, service);
+	auto iter = resolver.resolve(query);
+	decltype(iter) end;
+	std::vector<NetworkAddress> addrs;
+	while (iter != end) {
+		auto endpoint = iter->endpoint();
+		auto addr = endpoint.address();
+		if (addr.is_v6()) {
+			addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
+		} else {
+			addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
+		}
+		++iter;
+	}
+	return addrs;
+}
+
+Future<std::vector<NetworkAddress>> SimExternalConnection::resolveTCPEndpoint(const std::string& host,
+                                                                              const std::string& service) {
+	return resolveTCPEndpointImpl(host, service);
+}
+
+Future<Reference<IConnection>> SimExternalConnection::connect(NetworkAddress toAddr) {
+	return SimExternalConnectionImpl::connect(toAddr);
+}
+
+SimExternalConnection::SimExternalConnection(ip::tcp::socket&& socket)
+  : socket(std::move(socket)), dbgid(deterministicRandom()->randomUniqueID()) {}
+
+static constexpr auto testEchoServerPort = 8000;
+
+static void testEchoServer() {
+	static constexpr auto readBufferSize = 1000;
+	io_service ios;
+	ip::tcp::acceptor acceptor(ios, ip::tcp::endpoint(ip::tcp::v4(), testEchoServerPort));
+	ip::tcp::socket socket(ios);
+	acceptor.accept(socket);
+	loop {
+		char readBuffer[readBufferSize];
+		boost::system::error_code err;
+		auto length = socket.read_some(mutable_buffers_1(readBuffer, readBufferSize), err);
+		if (err == boost::asio::error::eof) {
+			return;
+		}
+		ASSERT(!err);
+		write(socket, buffer(readBuffer, length));
+	}
+}
+
+TEST_CASE("fdbrpc/SimExternalClient") {
+	state const size_t maxDataLength = 10000;
+	state std::thread serverThread([] { return testEchoServer(); });
+	state UnsentPacketQueue packetQueue;
+	state Reference<IConnection> externalConn;
+	loop {
+		Reference<IConnection> _externalConn =
+		    wait(INetworkConnections::net()->connect("localhost", std::to_string(testEchoServerPort)));
+		if (_externalConn.isValid()) {
+			externalConn = std::move(_externalConn);
+			break;
+		}
+		// Wait until server is ready
+		threadSleep(0.01);
+	}
+	state Key data = deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, maxDataLength + 1));
+	PacketWriter packetWriter(packetQueue.getWriteBuffer(data.size()), nullptr, Unversioned());
+	packetWriter.serializeBytes(data);
+	wait(externalConn->onWritable());
+	externalConn->write(packetQueue.getUnsent());
+	wait(externalConn->onReadable());
+	std::vector<uint8_t> vec(data.size());
+	externalConn->read(&vec[0], &vec[0] + vec.size());
+	externalConn->close();
+	StringRef echo(&vec[0], vec.size());
+	ASSERT(echo.toString() == data.toString());
+	serverThread.join();
+	return Void();
+}
+
+void forceLinkSimExternalConnectionTests() {}

fdbrpc/SimExternalConnection.h

@@ -0,0 +1,55 @@
+/*
+ * SimExternalConnection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBRPC_SIM_EXTERNAL_CONNECTION_H
+#define FDBRPC_SIM_EXTERNAL_CONNECTION_H
+#pragma once
+
+#include "flow/FastRef.h"
+#include "flow/network.h"
+#include "flow/flow.h"
+
+#include <boost/asio.hpp>
+
+class SimExternalConnection final : public IConnection, public ReferenceCounted<SimExternalConnection> {
+	boost::asio::ip::tcp::socket socket;
+	SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
+	UID dbgid;
+	std::deque<uint8_t> readBuffer;
+	AsyncTrigger onReadableTrigger;
+	friend class SimExternalConnectionImpl;
+
+public:
+	void addref() override { return ReferenceCounted<SimExternalConnection>::addref(); }
+	void delref() override { return ReferenceCounted<SimExternalConnection>::delref(); }
+	void close() override;
+	Future<Void> acceptHandshake() override;
+	Future<Void> connectHandshake() override;
+	Future<Void> onWritable() override;
+	Future<Void> onReadable() override;
+	int read(uint8_t* begin, uint8_t* end) override;
+	int write(SendBuffer const* buffer, int limit) override;
+	NetworkAddress getPeerAddress() const override;
+	UID getDebugID() const override;
+	static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host, const std::string& service);
+	static Future<Reference<IConnection>> connect(NetworkAddress toAddr);
+};
+
+#endif

fdbrpc/Stats.h

@@ -86,15 +86,15 @@ public:
 	void operator += (Value delta);
 	void operator ++ () { *this += 1; }
 	void clear();
-	void resetInterval();
+	void resetInterval() override;
 
-	std::string const& getName() const { return name; }
+	std::string const& getName() const override { return name; }
 
 	Value getIntervalDelta() const { return interval_delta; }
-	Value getValue() const { return interval_start_value + interval_delta; }
+	Value getValue() const override { return interval_start_value + interval_delta; }
 
 	// dValue / dt
-	double getRate() const;
+	double getRate() const override;
 
 	// Measures the clumpiness or dispersion of the counter.
 	// Computed as a normalized variance of the time between each incrementation of the value.
@@ -106,10 +106,10 @@ public:
 	// A uniformly periodic counter will have roughness of 0
 	// A uniformly periodic counter that increases in clumps of N will have roughness of N-1
 	// A counter with exponentially distributed incrementations will have roughness of 1
-	double getRoughness() const;
+	double getRoughness() const override;
 
-	bool hasRate() const { return true; }
+	bool hasRate() const override { return true; }
-	bool hasRoughness() const { return true; }
+	bool hasRoughness() const override { return true; }
 
 private:
 	std::string name;
@@ -152,15 +152,16 @@ Future<Void> traceCounters(std::string const& traceEventName, UID const& traceEv
 
 class LatencyBands {
 public:
-	LatencyBands(std::string name, UID id, double loggingInterval) : name(name), id(id), loggingInterval(loggingInterval), cc(nullptr), filteredCount(nullptr) {}
+	LatencyBands(std::string name, UID id, double loggingInterval)
+	  : name(name), id(id), loggingInterval(loggingInterval) {}
 
 	void addThreshold(double value) {
 		if(value > 0 && bands.count(value) == 0) {
 			if(bands.size() == 0) {
 				ASSERT(!cc && !filteredCount);
-				cc = new CounterCollection(name, id.toString());
+				cc = std::make_unique<CounterCollection>(name, id.toString());
-				logger = traceCounters(name, id, loggingInterval, cc, id.toString() + "/" + name);
+				logger = traceCounters(name, id, loggingInterval, cc.get(), id.toString() + "/" + name);
-				filteredCount = new Counter("Filtered", *cc);
+				filteredCount = std::make_unique<Counter>("Filtered", *cc);
 				insertBand(std::numeric_limits<double>::infinity());
 			}
 
@@ -181,18 +182,9 @@ public:
 
 	void clearBands() {
 		logger = Void();
-
-		for(auto itr : bands) {
-			delete itr.second;
-		}
-		
 		bands.clear();
-
+		filteredCount.reset();
-		delete filteredCount;
+		cc.reset();
-		delete cc;
-
-		filteredCount = nullptr;
-		cc = nullptr;
 	}
 
 	~LatencyBands() {
@@ -200,18 +192,18 @@ public:
 	}
 
 private:
-	std::map<double, Counter*> bands;
+	std::map<double, std::unique_ptr<Counter>> bands;
-	Counter *filteredCount;
+	std::unique_ptr<Counter> filteredCount;
 
 	std::string name;
 	UID id;
 	double loggingInterval;
 
-	CounterCollection *cc;
+	std::unique_ptr<CounterCollection> cc;
 	Future<Void> logger;
 
 	void insertBand(double value) {
-		bands.insert(std::make_pair(value, new Counter(format("Band%f", value), *cc)));
+		bands.emplace(std::make_pair(value, std::make_unique<Counter>(format("Band%f", value), *cc)));
 	}
 };
 

fdbrpc/fdbrpc.h

@@ -111,7 +111,7 @@ struct NetSAV final : SAV<T>, FlowReceiver, FastAllocated<NetSAV<T>> {
 };
 
 template <class T>
-class ReplyPromise sealed : public ComposedIdentifier<T, 1> {
+class ReplyPromise final : public ComposedIdentifier<T, 1> {
 public:
 	template <class U>
 	void send(U&& value) const {

fdbrpc/sim2.actor.cpp

@@ -24,6 +24,10 @@
 #include <vector>
 
 #include "fdbrpc/simulator.h"
+#define BOOST_SYSTEM_NO_LIB
+#define BOOST_DATE_TIME_NO_LIB
+#define BOOST_REGEX_NO_LIB
+#include "fdbrpc/SimExternalConnection.h"
 #include "flow/ActorCollection.h"
 #include "flow/IRandom.h"
 #include "flow/IThreadPool.h"
@@ -96,18 +100,9 @@ void ISimulator::displayWorkers() const
 int openCount = 0;
 
 struct SimClogging {
-	double getSendDelay( NetworkAddress from, NetworkAddress to ) {
+	double getSendDelay(NetworkAddress from, NetworkAddress to) const { return halfLatency(); }
-		return halfLatency();
-		double tnow = now();
-		double t = tnow + halfLatency();
 
-		if (!g_simulator.speedUpSimulation && clogSendUntil.count( to.ip ))
+	double getRecvDelay(NetworkAddress from, NetworkAddress to) {
-			t = std::max( t, clogSendUntil[ to.ip ] );
-
-		return t - tnow;
-	}
-
-	double getRecvDelay( NetworkAddress from, NetworkAddress to ) {
 		auto pair = std::make_pair( from.ip, to.ip );
 
 		double tnow = now();
@@ -501,9 +496,7 @@ public:
 
 	std::string getFilename() const override { return actualFilename; }
 
-	~SimpleFile() {
+	~SimpleFile() override { _close(h); }
-		_close( h );
-	}
 
 private:
 	int h;
@@ -795,7 +788,7 @@ public:
 	TaskPriority getCurrentTask() const override { return currentTaskID; }
 	void setCurrentTask(TaskPriority taskID) override { currentTaskID = taskID; }
 	// Sets the taskID/priority of the current task, without yielding
-	Future<Reference<IConnection>> connect(NetworkAddress toAddr, std::string host) override {
+	Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string &host) override {
 		ASSERT( host.empty());
 		if (!addressMap.count( toAddr )) {
 			return waitForProcessAndConnect( toAddr, this );
@@ -820,11 +813,15 @@ public:
 		return onConnect( ::delay(0.5*deterministicRandom()->random01()), myc );
 	}
 
+	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string &host) override {
+		return SimExternalConnection::connect(toAddr);
+	}
+
 	Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) override;
 	Future<Reference<IUDPSocket>> createUDPSocket(bool isV6 = false) override;
 
-  Future<std::vector<NetworkAddress>> resolveTCPEndpoint(std::string host, std::string service) override {
+	Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string &host, const std::string &service) override {
-		throw lookup_failed();
+		return SimExternalConnection::resolveTCPEndpoint(host, service);
 	}
 	ACTOR static Future<Reference<IConnection>> onConnect( Future<Void> ready, Reference<Sim2Conn> conn ) {
 		wait(ready);
@@ -1803,7 +1800,7 @@ public:
 		ASSERT(process->boundUDPSockets.find(localAddress) == process->boundUDPSockets.end());
 		process->boundUDPSockets.emplace(localAddress, this);
 	}
-	~UDPSimSocket() {
+	~UDPSimSocket() override {
 		if (!closed.getFuture().isReady()) {
 			close();
 			closed.send(Void());
@@ -1900,6 +1897,10 @@ public:
 		return _localAddress;
 	}
 
+	boost::asio::ip::udp::socket::native_handle_type native_handle() override {
+		return 0;
+	}
+
 };
 
 Future<Reference<IUDPSocket>> Sim2::createUDPSocket(NetworkAddress toAddr) {
@@ -2036,8 +2037,7 @@ int sf_open( const char* filename, int flags, int convFlags, int mode ) {
 #endif
 
 // Opens a file for asynchronous I/O
-Future< Reference<class IAsyncFile> > Sim2FileSystem::open( std::string filename, int64_t flags, int64_t mode )
+Future<Reference<class IAsyncFile>> Sim2FileSystem::open(const std::string& filename, int64_t flags, int64_t mode) {
-{
 	ASSERT( (flags & IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE) ||
 			!(flags & IAsyncFile::OPEN_CREATE) ||
 			StringRef(filename).endsWith(LiteralStringRef(".fdb-lock")) );  // We don't use "ordinary" non-atomic file creation right now except for folder locking, and we don't have code to simulate its unsafeness.
@@ -2074,12 +2074,11 @@ Future< Reference<class IAsyncFile> > Sim2FileSystem::open( std::string filename
 }
 
 // Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-Future< Void > Sim2FileSystem::deleteFile( std::string filename, bool mustBeDurable )
+Future<Void> Sim2FileSystem::deleteFile(const std::string& filename, bool mustBeDurable) {
-{
 	return Sim2::deleteFileImpl(&g_sim2, filename, mustBeDurable);
 }
 
-Future< std::time_t > Sim2FileSystem::lastWriteTime( std::string filename ) {
+Future<std::time_t> Sim2FileSystem::lastWriteTime(const std::string& filename) {
 	// TODO: update this map upon file writes.
 	static std::map<std::string, double> fileWrites;
 	if (BUGGIFY && deterministicRandom()->random01() < 0.01) {

fdbrpc/simulator.h

@@ -36,12 +36,16 @@ enum ClogMode { ClogDefault, ClogAll, ClogSend, ClogReceive };
 
 class ISimulator : public INetwork {
 public:
-	ISimulator() : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), isStopped(false), lastConnectionFailure(0), connectionFailuresDisableDuration(0), speedUpSimulation(false), allSwapsDisabled(false), backupAgents(WaitForType), drAgents(WaitForType), extraDB(nullptr), allowLogSetKills(true), usableRegions(1) {}
+	ISimulator()
+	  : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1),
+	    isStopped(false), lastConnectionFailure(0), connectionFailuresDisableDuration(0), speedUpSimulation(false),
+	    allSwapsDisabled(false), backupAgents(BackupAgentType::WaitForType), drAgents(BackupAgentType::WaitForType), extraDB(nullptr),
+	    allowLogSetKills(true), usableRegions(1) {}
 
 	// Order matters!
 	enum KillType { KillInstantly, InjectFaults, RebootAndDelete, RebootProcessAndDelete, Reboot, RebootProcess, None };
 
-	enum BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
+	enum class BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
 
 	// Subclasses may subclass ProcessInfo as well
 	struct MachineInfo;
@@ -89,7 +93,7 @@ public:
 		bool isAvailable() const { return !isExcluded() && isReliable(); }
 		bool isExcluded() const { return excluded; }
 		bool isCleared() const { return cleared; }
-		std::string getReliableInfo() {
+		std::string getReliableInfo() const {
 			std::stringstream ss;
 			ss << "failed:" << failed << " fault_injection_p1:" << fault_injection_p1
 			   << " fault_injection_p2:" << fault_injection_p2;
@@ -123,7 +127,7 @@ public:
 			}
 		}
 
-		const Reference<IListener> getListener(const NetworkAddress& addr) {
+		Reference<IListener> getListener(const NetworkAddress& addr) const {
 			auto listener = listenerMap.find(addr);
 			ASSERT( listener != listenerMap.end());
 			return listener->second;
@@ -153,7 +157,7 @@ public:
 		std::set<std::string> closingFiles;
 		Optional<Standalone<StringRef>>	machineId;
 
-		MachineInfo() : machineProcess(0) {}
+		MachineInfo() : machineProcess(nullptr) {}
 	};
 
 	ProcessInfo* getProcess( Endpoint const& endpoint ) { return getProcessByAddress(endpoint.getPrimaryAddress()); }
@@ -178,15 +182,13 @@ public:
 	virtual bool isAvailable() const = 0;
 	virtual bool datacenterDead(Optional<Standalone<StringRef>> dcId) const = 0;
 	virtual void displayWorkers() const;
-
+	ProtocolVersion protocolVersion() override = 0;
-	virtual ProtocolVersion protocolVersion() = 0;
+	void addRole(NetworkAddress const& address, std::string const& role) {
-
-	virtual void addRole(NetworkAddress const& address, std::string const& role) {
 		roleAddresses[address][role] ++;
 		TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("NumRoles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
 	}
 
-	virtual void removeRole(NetworkAddress const& address, std::string const& role) {
+	void removeRole(NetworkAddress const& address, std::string const& role) {
 		auto addressIt = roleAddresses.find(address);
 		if (addressIt != roleAddresses.end()) {
 			auto rolesIt = addressIt->second.find(role);
@@ -215,7 +217,7 @@ public:
 		}
 	}
 
-	virtual std::string getRoles(NetworkAddress const& address, bool skipWorkers = true) const {
+	std::string getRoles(NetworkAddress const& address, bool skipWorkers = true) const {
 		auto addressIt = roleAddresses.find(address);
 		std::string roleText;
 		if (addressIt != roleAddresses.end()) {
@@ -229,20 +231,20 @@ public:
 		return roleText;
 	}
 
-	virtual void clearAddress(NetworkAddress const& address) {
+	void clearAddress(NetworkAddress const& address) {
 		clearedAddresses[address]++;
 		TraceEvent("ClearAddress").detail("Address", address).detail("Value", clearedAddresses[address]);
 	}
-	virtual bool isCleared(NetworkAddress const& address) const {
+	bool isCleared(NetworkAddress const& address) const {
 		return clearedAddresses.find(address) != clearedAddresses.end();
 	}
 
-	virtual void excludeAddress(NetworkAddress const& address) {
+	void excludeAddress(NetworkAddress const& address) {
 		excludedAddresses[address]++;
 		TraceEvent("ExcludeAddress").detail("Address", address).detail("Value", excludedAddresses[address]);
 	}
 
-	virtual void includeAddress(NetworkAddress const& address) {
+	void includeAddress(NetworkAddress const& address) {
 		auto addressIt = excludedAddresses.find(address);
 		if (addressIt != excludedAddresses.end()) {
 			if (addressIt->second > 1) {
@@ -258,29 +260,27 @@ public:
 			TraceEvent(SevWarn,"IncludeAddress").detail("Address", address).detail("Result", "Missing");
 		}
 	}
-	virtual void includeAllAddresses() {
+	void includeAllAddresses() {
 		TraceEvent("IncludeAddressAll").detail("AddressTotal", excludedAddresses.size());
 		excludedAddresses.clear();
 	}
-	virtual bool isExcluded(NetworkAddress const& address) const {
+	bool isExcluded(NetworkAddress const& address) const {
 		return excludedAddresses.find(address) != excludedAddresses.end();
 	}
 
-	virtual void disableSwapToMachine(Optional<Standalone<StringRef>> zoneId ) {
+	void disableSwapToMachine(Optional<Standalone<StringRef>> zoneId) { swapsDisabled.insert(zoneId); }
-		swapsDisabled.insert(zoneId);
+	void enableSwapToMachine(Optional<Standalone<StringRef>> zoneId) {
-	}
-	virtual void enableSwapToMachine(Optional<Standalone<StringRef>> zoneId ) {
 		swapsDisabled.erase(zoneId);
 		allSwapsDisabled = false;
 	}
-	virtual bool canSwapToMachine(Optional<Standalone<StringRef>> zoneId ) {
+	bool canSwapToMachine(Optional<Standalone<StringRef>> zoneId) const {
 		return swapsDisabled.count( zoneId ) == 0 && !allSwapsDisabled && !extraDB;
 	}
-	virtual void enableSwapsToAll() {
+	void enableSwapsToAll() {
 		swapsDisabled.clear();
 		allSwapsDisabled = false;
 	}
-	virtual void disableSwapsToAll() {
+	void disableSwapsToAll() {
 		swapsDisabled.clear();
 		allSwapsDisabled = true;
 	}
@@ -291,7 +291,7 @@ public:
 	virtual ProcessInfo* getProcessByAddress( NetworkAddress const& address ) = 0;
 	virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0;
 	virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& machineId) = 0;
-	virtual void run() {}
+	void run() override {}
 	virtual void destroyProcess( ProcessInfo *p ) = 0;
 	virtual void destroyMachine(Optional<Standalone<StringRef>> const& machineId ) = 0;
 
@@ -335,15 +335,12 @@ public:
 	bool hasDiffProtocolProcess; // true if simulator is testing a process with a different version
 	bool setDiffProtocol; // true if a process with a different protocol version has been started
 
-	virtual flowGlobalType global(int id) const { return getCurrentProcess()->global(id); };
+	flowGlobalType global(int id) const final { return getCurrentProcess()->global(id); };
-	virtual void setGlobal(size_t id, flowGlobalType v) { getCurrentProcess()->setGlobal(id,v); };
+	void setGlobal(size_t id, flowGlobalType v) final { getCurrentProcess()->setGlobal(id, v); };
 
-	virtual void disableFor(const std::string& desc, double time) {
+	void disableFor(const std::string& desc, double time) { disabledMap[desc] = time; }
-		disabledMap[desc] = time;
-	}
 
-	virtual double checkDisabled(const std::string& desc) const
+	double checkDisabled(const std::string& desc) const {
-	{
 		auto iter = disabledMap.find(desc);
 		if (iter != disabledMap.end()) {
 			return iter->second;
@@ -386,16 +383,16 @@ extern Future<Void> waitUntilDiskReady(Reference<DiskParameters> parameters, int
 class Sim2FileSystem : public IAsyncFileSystem {
 public:
 	// Opens a file for asynchronous I/O
-	virtual Future< Reference<class IAsyncFile> > open( std::string filename, int64_t flags, int64_t mode );
+	Future<Reference<class IAsyncFile>> open(const std::string& filename, int64_t flags, int64_t mode) override;
 
 	// Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
-	virtual Future< Void > deleteFile( std::string filename, bool mustBeDurable );
+	Future<Void> deleteFile(const std::string& filename, bool mustBeDurable) override;
 
-	virtual Future< std::time_t > lastWriteTime( std::string filename );
+	Future<std::time_t> lastWriteTime(const std::string& filename) override;
 
 	Sim2FileSystem() {}
 
-	virtual ~Sim2FileSystem() {}
+	~Sim2FileSystem() override {}
 
 	static void newFileSystem();
 };

fdbserver/BackupProgress.actor.h

@@ -70,7 +70,7 @@ public:
 	void delref() { ReferenceCounted<BackupProgress>::delref(); }
 
 private:
-	std::set<Tag> enumerateLogRouterTags(int logRouterTags) {
+	std::set<Tag> enumerateLogRouterTags(int logRouterTags) const {
 		std::set<Tag> tags;
 		for (int i = 0; i < logRouterTags; i++) {
 			tags.insert(Tag(tagLocalityLogRouter, i));

fdbserver/CMakeLists.txt

@@ -48,7 +48,7 @@ set(FDBSERVER_SRCS
   LogSystemPeekCursor.actor.cpp
   MasterInterface.h
   MetricLogger.actor.cpp
-  MetricLogger.h
+  MetricLogger.actor.h
   CommitProxyServer.actor.cpp
   masterserver.actor.cpp
   MutationTracking.h
@@ -105,6 +105,7 @@ set(FDBSERVER_SRCS
   TLogInterface.h
   TLogServer.actor.cpp
   VersionedBTree.actor.cpp
+  VFSAsync.h
   VFSAsync.cpp
   WaitFailure.actor.cpp
   WaitFailure.h
@@ -119,7 +120,6 @@ set(FDBSERVER_SRCS
   workloads/AsyncFileRead.actor.cpp
   workloads/AsyncFileWrite.actor.cpp
   workloads/AtomicOps.actor.cpp
-  workloads/ReadHotDetection.actor.cpp
   workloads/AtomicOpsApiCorrectness.actor.cpp
   workloads/AtomicRestore.actor.cpp
   workloads/AtomicSwitchover.actor.cpp
@@ -131,6 +131,7 @@ set(FDBSERVER_SRCS
   workloads/BackupToDBAbort.actor.cpp
   workloads/BackupToDBCorrectness.actor.cpp
   workloads/BackupToDBUpgrade.actor.cpp
+  workloads/BlobStoreWorkload.h
   workloads/BulkLoad.actor.cpp
   workloads/BulkSetup.actor.h
   workloads/Cache.actor.cpp
@@ -185,6 +186,7 @@ set(FDBSERVER_SRCS
   workloads/RandomMoveKeys.actor.cpp
   workloads/RandomSelector.actor.cpp
   workloads/ReadAfterWrite.actor.cpp
+  workloads/ReadHotDetection.actor.cpp
   workloads/ReadWrite.actor.cpp
   workloads/RemoveServersSafely.actor.cpp
   workloads/ReportConflictingKeys.actor.cpp
@@ -280,9 +282,11 @@ if (GPERFTOOLS_FOUND)
   target_link_libraries(fdbserver PRIVATE gperftools)
 endif()
 
-if(GENERATE_DEBUG_PACKAGES)
+if(NOT OPEN_FOR_IDE)
-  fdb_install(TARGETS fdbserver DESTINATION sbin COMPONENT server)
+  if(GENERATE_DEBUG_PACKAGES)
-else()
+    fdb_install(TARGETS fdbserver DESTINATION sbin COMPONENT server)
-  add_custom_target(prepare_fdbserver_install ALL DEPENDS strip_only_fdbserver)
+  else()
-  fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbserver DESTINATION sbin COMPONENT server)
+    add_custom_target(prepare_fdbserver_install ALL DEPENDS strip_only_fdbserver)
+    fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbserver DESTINATION sbin COMPONENT server)
+  endif()
 endif()