Merge branch 'main' into feature-metacluster

2022-06-28 14:58:07 -07:00 · 2022-06-28 14:58:07 -07:00 · e1a93988ef
parent 899b6267ae ec9c3cf05c
commit e1a93988ef
590 changed files with 4346 additions and 3399 deletions
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@ -0,0 +1,4 @@
+# clang-format the entire codebase
+df90cc89de67ea4748c8cadd18e6fc4ce7fda12e
+2c788c233db56ccec4ed90d7da31887487b9f3b7
+69508b980f3cc5aabea6322f292e53b07bb27544
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -105,15 +105,12 @@ set(FDB_PACKAGE_NAME "${FDB_MAJOR}.${FDB_MINOR}")
 configure_file(${CMAKE_SOURCE_DIR}/versions.target.cmake ${CMAKE_CURRENT_BINARY_DIR}/versions.target)
 file(WRITE ${CMAKE_BINARY_DIR}/version.txt ${FDB_VERSION})

-message(STATUS "FDB version is ${FDB_VERSION}")
-message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}")
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/fdbclient/versions.h)
-
-
 ################################################################################
 # Flow
 ################################################################################

+include(utils)
+
 # Flow and other tools are written in C# - so we need that dependency
 include(EnableCsharp)

@ -203,12 +200,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
  add_link_options(-lexecinfo)
 endif()

-################################################################################
-# Build information
-################################################################################
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/fdbclient/BuildFlags.h)
-
 ################################################################################
 # process compile commands for IDE
 ################################################################################
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@ -137,7 +137,6 @@ if(NOT WIN32)
    test/apitester/TesterUtil.h
    test/apitester/TesterWorkload.cpp
    test/apitester/TesterWorkload.h
-    ../../flow/SimpleOpt.h
  )

  if(OPEN_FOR_IDE)
@ -180,7 +179,7 @@ if(NOT WIN32)
  add_dependencies(disconnected_timeout_unit_tests doctest)
  target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
  target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
-  target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
+  target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/flow/include)
  target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
  target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient rapidjson)
@ -188,18 +187,20 @@ if(NOT WIN32)
  target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)
  target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)

-if(USE_SANITIZER)
-  target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
-else()
-  target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
-endif()
+  if(USE_SANITIZER)
+    target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
+  else()
+    target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
+  endif()
+  target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_SOURCE_DIR}/flow/include" "${CMAKE_BINARY_DIR}/flow/include")
+  target_link_libraries(fdb_c_api_tester PRIVATE SimpleOpt)

  # do not set RPATH for mako
  set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
  if (USE_SANITIZER)
-	target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_asan)
+    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_asan rapidjson)
  else()
-	target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target)
+    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
  endif()

  if(NOT OPEN_FOR_IDE)
@ -330,7 +331,7 @@ endif()
      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
            --build-dir ${CMAKE_BINARY_DIR}
            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-            --upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0"
+            --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0"
            --process-number 1
            )

@ -338,7 +339,7 @@ endif()
      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
            --build-dir ${CMAKE_BINARY_DIR}
            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-            --upgrade-path "7.0.0" "7.1.5" "7.2.0"
+            --upgrade-path "7.0.0" "7.1.9" "7.2.0"
            --process-number 1
            )

@ -346,7 +347,7 @@ endif()
        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
            --build-dir ${CMAKE_BINARY_DIR}
            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-            --upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0" "7.1.5"
+            --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0" "7.1.9"
            --process-number 3
          )

@ -354,7 +355,7 @@ endif()
        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
            --build-dir ${CMAKE_BINARY_DIR}
            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-            --upgrade-path "7.0.0" "7.1.5" "7.2.0" "7.1.5"
+            --upgrade-path "7.0.0" "7.1.9" "7.2.0" "7.1.9"
            --process-number 3
          )

@ -362,7 +363,7 @@ endif()
        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
            --build-dir ${CMAKE_BINARY_DIR}
            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-            --upgrade-path "7.1.5" "7.2.0" "7.1.5"
+            --upgrade-path "7.1.9" "7.2.0" "7.1.9"
            --process-number 3
          )

@ -376,15 +377,25 @@ endif()
              --redundancy double
            )

-    add_test(NAME fdb_c_wiggle_and_upgrade
-          COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "7.0.0" "wiggle" "7.2.0"
-              --disable-log-dump
-              --process-number 3
-              --redundancy double
-            )
+    add_test(NAME fdb_c_wiggle_and_upgrade_latest
+      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+          --build-dir ${CMAKE_BINARY_DIR}
+          --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+          --upgrade-path "7.1.9" "wiggle" "7.2.0"
+          --disable-log-dump
+          --process-number 3
+          --redundancy double
+        )
+
+      add_test(NAME fdb_c_wiggle_and_upgrade_63
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+            --upgrade-path "6.3.24" "wiggle" "7.0.0"
+            --disable-log-dump
+            --process-number 3
+            --redundancy double
+          )

  endif()

--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@ -501,6 +501,28 @@ extern "C" DLLEXPORT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant
 	CATCH_AND_RETURN(*out_transaction = (FDBTransaction*)TENANT(tenant)->createTransaction().extractPtr(););
 }

+extern "C" DLLEXPORT FDBFuture* fdb_tenant_purge_blob_granules(FDBTenant* tenant,
+                                                               uint8_t const* begin_key_name,
+                                                               int begin_key_name_length,
+                                                               uint8_t const* end_key_name,
+                                                               int end_key_name_length,
+                                                               int64_t purge_version,
+                                                               fdb_bool_t force) {
+	return (FDBFuture*)(TENANT(tenant)
+	                        ->purgeBlobGranules(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
+	                                                        StringRef(end_key_name, end_key_name_length)),
+	                                            purge_version,
+	                                            force)
+	                        .extractPtr());
+}
+extern "C" DLLEXPORT FDBFuture* fdb_tenant_wait_purge_granules_complete(FDBTenant* tenant,
+                                                                        uint8_t const* purge_key_name,
+                                                                        int purge_key_name_length) {
+	return (FDBFuture*)(TENANT(tenant)
+	                        ->waitPurgeGranulesComplete(StringRef(purge_key_name, purge_key_name_length))
+	                        .extractPtr());
+}
+
 extern "C" DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant) {
 	try {
 		TENANT(tenant)->delref();
@ -684,27 +706,6 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_mapped_range(FDBTransaction*
 	                    .extractPtr());
 }

-FDBFuture* fdb_transaction_get_range_and_flat_map_v709(FDBTransaction* tr,
-                                                       uint8_t const* begin_key_name,
-                                                       int begin_key_name_length,
-                                                       fdb_bool_t begin_or_equal,
-                                                       int begin_offset,
-                                                       uint8_t const* end_key_name,
-                                                       int end_key_name_length,
-                                                       fdb_bool_t end_or_equal,
-                                                       int end_offset,
-                                                       uint8_t const* mapper_name,
-                                                       int mapper_name_length,
-                                                       int limit,
-                                                       int target_bytes,
-                                                       FDBStreamingMode mode,
-                                                       int iteration,
-                                                       fdb_bool_t snapshot,
-                                                       fdb_bool_t reverse) {
-	fprintf(stderr, "GetRangeAndFlatMap is removed from 7.0. Please upgrade to 7.1 and use GetMappedRange\n");
-	abort();
-}
-
 FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr,
                                                  uint8_t const* begin_key_name,
                                                  int begin_key_name_length,
@ -938,7 +939,6 @@ extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version
 	// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to
 	// undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public
 	// functions call an internal implementation function. See fdb_create_database_impl for an example.
-	FDB_API_REMOVED(fdb_transaction_get_range_and_flat_map, 710);
 	FDB_API_REMOVED(fdb_future_get_version, 620);
 	FDB_API_REMOVED(fdb_create_cluster, 610);
 	FDB_API_REMOVED(fdb_cluster_create_database, 610);
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@ -315,6 +315,18 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_wait_purge_granules_complet
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant,
                                                                       FDBTransaction** out_transaction);

+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_purge_blob_granules(FDBTenant* db,
+                                                                       uint8_t const* begin_key_name,
+                                                                       int begin_key_name_length,
+                                                                       uint8_t const* end_key_name,
+                                                                       int end_key_name_length,
+                                                                       int64_t purge_version,
+                                                                       fdb_bool_t force);
+
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_wait_purge_granules_complete(FDBTenant* db,
+                                                                                uint8_t const* purge_key_name,
+                                                                                int purge_key_name_length);
+
 DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant);

 DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr);
--- a/bindings/c/test/apitester/fdb_c_api_tester.cpp
+++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp
@ -24,7 +24,7 @@
 #include "TesterTransactionExecutor.h"
 #include "TesterTestSpec.h"
 #include "TesterUtil.h"
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "test/fdb_api.hpp"

 #include <memory>
--- a/bindings/c/test/mako/ddsketch.hpp
+++ b/bindings/c/test/mako/ddsketch.hpp
@ -118,7 +118,7 @@ public:
 			return T(0);

 		int index = -1;
-		bool found = false;
+		[[maybe_unused]] bool found = false;
 		if (percentile <= 0.5) { // count up
 			uint64_t count = zeroPopulationSize;
 			for (size_t i = 0; i < buckets.size(); i++) {
@ -272,4 +272,4 @@ private:
 	constexpr static double gamma = 1.189207115;
 };

-#endif
+#endif
--- a/bindings/c/test/mako/mako.cpp
+++ b/bindings/c/test/mako/mako.cpp
@ -87,7 +87,7 @@ Transaction createNewTransaction(Database db, Arguments const& args, int id = -1
 	}
 	// Create Tenant Transaction
 	int tenant_id = (id == -1) ? urand(0, args.tenants - 1) : id;
-	// If provided tenants array (only necessary in runWorkload), use it
+	// If provided tenants array, use it
 	if (tenants) {
 		return tenants[tenant_id].createTransaction();
 	}
@ -167,8 +167,6 @@ int populate(Database db,
             int thread_id,
             int thread_tps,
             ThreadStatistics& stats) {
-	const auto key_begin = insertBegin(args.rows, worker_id, thread_id, args.num_processes, args.num_threads);
-	const auto key_end = insertEnd(args.rows, worker_id, thread_id, args.num_processes, args.num_threads);
 	auto xacts = 0;

 	auto keystr = ByteString{};
@ -181,7 +179,6 @@ int populate(Database db,
 	auto watch_throttle = Stopwatch(watch_total.getStart());
 	auto watch_tx = Stopwatch(watch_total.getStart());
 	auto watch_trace = Stopwatch(watch_total.getStart());
-	auto key_checkpoint = key_begin; // in case of commit failure, restart from this key

 	Transaction systemTx = db.createTransaction();
 	for (int i = 0; i < args.tenants; ++i) {
@ -204,77 +201,92 @@ int populate(Database db,
 			}
 		}
 	}
-	Transaction tx = createNewTransaction(db, args);
-	for (auto i = key_begin; i <= key_end; i++) {
-		/* sequential keys */
-		genKey(keystr.data(), KEY_PREFIX, args, i);
-		/* random values */
-		randomString(valstr.data(), args.value_length);
-
-		while (thread_tps > 0 && xacts >= thread_tps /* throttle */) {
-			if (toIntegerSeconds(watch_throttle.stop().diff()) >= 1) {
-				xacts = 0;
-				watch_throttle.startFromStop();
-			} else {
-				usleep(1000);
-			}
-		}
-		if (num_seconds_trace_every) {
-			if (toIntegerSeconds(watch_trace.stop().diff()) >= num_seconds_trace_every) {
-				watch_trace.startFromStop();
-				logr.debug("txn tracing {}", toCharsRef(keystr));
-				auto err = Error{};
-				err = tx.setOptionNothrow(FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, keystr);
-				if (err) {
-					logr.error("setOption(TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER): {}", err.what());
-				}
-				err = tx.setOptionNothrow(FDB_TR_OPTION_LOG_TRANSACTION, BytesRef());
-				if (err) {
-					logr.error("setOption(TR_OPTION_LOG_TRANSACTION): {}", err.what());
-				}
-			}
-		}
-
-		/* insert (SET) */
-		tx.set(keystr, valstr);
-		stats.incrOpCount(OP_INSERT);
-
-		/* commit every 100 inserts (default) or if this is the last key */
-		if ((i % num_commit_every == 0) || i == key_end) {
-			const auto do_sample = (stats.getOpCount(OP_TRANSACTION) % args.sampling) == 0;
-			auto watch_commit = Stopwatch(StartAtCtor{});
-			auto future_commit = tx.commit();
-			const auto rc = waitAndHandleError(tx, future_commit, "COMMIT_POPULATE_INSERT");
-			watch_commit.stop();
-			watch_tx.setStop(watch_commit.getStop());
-			auto tx_restarter = ExitGuard([&watch_tx]() { watch_tx.startFromStop(); });
-			if (rc == FutureRC::OK) {
-				key_checkpoint = i + 1; // restart on failures from next key
-				tx = createNewTransaction(db, args);
-			} else if (rc == FutureRC::ABORT) {
-				return -1;
-			} else {
-				i = key_checkpoint - 1; // restart from last committed
-				continue;
-			}
-			/* xact latency stats */
-			if (do_sample) {
-				const auto commit_latency = watch_commit.diff();
-				const auto tx_duration = watch_tx.diff();
-				stats.addLatency(OP_COMMIT, commit_latency);
-				stats.addLatency(OP_TRANSACTION, tx_duration);
-			}
-			stats.incrOpCount(OP_COMMIT);
-			stats.incrOpCount(OP_TRANSACTION);
-
-			xacts++; /* for throttling */
-		}
+	// mimic typical tenant usage: keep tenants in memory
+	// and create transactions as needed
+	Tenant tenants[args.tenants];
+	for (int i = 0; i < args.tenants; ++i) {
+		std::string tenantStr = "tenant" + std::to_string(i);
+		BytesRef tenant_name = toBytesRef(tenantStr);
+		tenants[i] = db.openTenant(tenant_name);
+	}
+	int populate_iters = args.tenants > 0 ? args.tenants : 1;
+	// Each tenant should have the same range populated
+	for (auto t_id = 0; t_id < populate_iters; ++t_id) {
+		Transaction tx = createNewTransaction(db, args, t_id, args.tenants > 0 ? tenants : nullptr);
+		const auto key_begin = insertBegin(args.rows, worker_id, thread_id, args.num_processes, args.num_threads);
+		const auto key_end = insertEnd(args.rows, worker_id, thread_id, args.num_processes, args.num_threads);
+		auto key_checkpoint = key_begin; // in case of commit failure, restart from this key
+		for (auto i = key_begin; i <= key_end; i++) {
+			/* sequential keys */
+			genKey(keystr.data(), KEY_PREFIX, args, i);
+			/* random values */
+			randomString(valstr.data(), args.value_length);
+
+			while (thread_tps > 0 && xacts >= thread_tps /* throttle */) {
+				if (toIntegerSeconds(watch_throttle.stop().diff()) >= 1) {
+					xacts = 0;
+					watch_throttle.startFromStop();
+				} else {
+					usleep(1000);
+				}
+			}
+			if (num_seconds_trace_every) {
+				if (toIntegerSeconds(watch_trace.stop().diff()) >= num_seconds_trace_every) {
+					watch_trace.startFromStop();
+					logr.debug("txn tracing {}", toCharsRef(keystr));
+					auto err = Error{};
+					err = tx.setOptionNothrow(FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, keystr);
+					if (err) {
+						logr.error("setOption(TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER): {}", err.what());
+					}
+					err = tx.setOptionNothrow(FDB_TR_OPTION_LOG_TRANSACTION, BytesRef());
+					if (err) {
+						logr.error("setOption(TR_OPTION_LOG_TRANSACTION): {}", err.what());
+					}
+				}
+			}
+
+			/* insert (SET) */
+			tx.set(keystr, valstr);
+			stats.incrOpCount(OP_INSERT);
+
+			/* commit every 100 inserts (default) or if this is the last key */
+			if ((i % num_commit_every == 0) || i == key_end) {
+				const auto do_sample = (stats.getOpCount(OP_TRANSACTION) % args.sampling) == 0;
+				auto watch_commit = Stopwatch(StartAtCtor{});
+				auto future_commit = tx.commit();
+				const auto rc = waitAndHandleError(tx, future_commit, "COMMIT_POPULATE_INSERT");
+				watch_commit.stop();
+				watch_tx.setStop(watch_commit.getStop());
+				auto tx_restarter = ExitGuard([&watch_tx]() { watch_tx.startFromStop(); });
+				if (rc == FutureRC::OK) {
+					key_checkpoint = i + 1; // restart on failures from next key
+					tx = createNewTransaction(db, args, t_id, args.tenants > 0 ? tenants : nullptr);
+				} else if (rc == FutureRC::ABORT) {
+					return -1;
+				} else {
+					i = key_checkpoint - 1; // restart from last committed
+					continue;
+				}
+				/* xact latency stats */
+				if (do_sample) {
+					const auto commit_latency = watch_commit.diff();
+					const auto tx_duration = watch_tx.diff();
+					stats.addLatency(OP_COMMIT, commit_latency);
+					stats.addLatency(OP_TRANSACTION, tx_duration);
+				}
+				stats.incrOpCount(OP_COMMIT);
+				stats.incrOpCount(OP_TRANSACTION);
+
+				xacts++; /* for throttling */
+			}
+		}
+		logr.debug("Populated {} rows [{}, {}]: {:6.3f} sec",
+		           key_end - key_begin + 1,
+		           key_begin,
+		           key_end,
+		           toDoubleSeconds(watch_total.stop().diff()));
 	}
-	logr.debug("Populated {} rows [{}, {}]: {:6.3f} sec",
-	           key_end - key_begin + 1,
-	           key_begin,
-	           key_end,
-	           toDoubleSeconds(watch_total.stop().diff()));

 	return 0;
 }
@ -2096,6 +2108,10 @@ int main(int argc, char* argv[]) {
 		/* usage printed */
 		return 0;
 	}
+	if (args.tenants > 1) {
+		args.rows = args.rows / args.tenants;
+		args.row_digits = digits(args.rows);
+	}

 	rc = validateArguments(args);
 	if (rc < 0)
--- a/bindings/c/test/mako/stats.hpp
+++ b/bindings/c/test/mako/stats.hpp
@ -35,10 +35,10 @@
 #include "operations.hpp"
 #include "time.hpp"
 #include "ddsketch.hpp"
-#include "contrib/rapidjson/rapidjson/document.h"
-#include "contrib/rapidjson/rapidjson/rapidjson.h"
-#include "contrib/rapidjson/rapidjson/stringbuffer.h"
-#include "contrib/rapidjson/rapidjson/writer.h"
+#include "rapidjson/document.h"
+#include "rapidjson/rapidjson.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
 #include <iostream>
 #include <sstream>
 #include <vector>
--- a/bindings/c/test/unit/fdb_api.cpp
+++ b/bindings/c/test/unit/fdb_api.cpp
@ -157,6 +157,25 @@ Tenant::Tenant(FDBDatabase* db, const uint8_t* name, int name_length) {
 	}
 }

+KeyFuture Tenant::purge_blob_granules(FDBTenant* tenant,
+                                      std::string_view begin_key,
+                                      std::string_view end_key,
+                                      int64_t purge_version,
+                                      fdb_bool_t force) {
+	return KeyFuture(fdb_tenant_purge_blob_granules(tenant,
+	                                                (const uint8_t*)begin_key.data(),
+	                                                begin_key.size(),
+	                                                (const uint8_t*)end_key.data(),
+	                                                end_key.size(),
+	                                                purge_version,
+	                                                force));
+}
+
+EmptyFuture Tenant::wait_purge_granules_complete(FDBTenant* tenant, std::string_view purge_key) {
+	return EmptyFuture(
+	    fdb_tenant_wait_purge_granules_complete(tenant, (const uint8_t*)purge_key.data(), purge_key.size()));
+}
+
 Tenant::~Tenant() {
 	if (tenant != nullptr) {
 		fdb_tenant_destroy(tenant);
--- a/bindings/c/test/unit/fdb_api.hpp
+++ b/bindings/c/test/unit/fdb_api.hpp
@ -98,6 +98,7 @@ public:
 private:
 	friend class Transaction;
 	friend class Database;
+	friend class Tenant;
 	KeyFuture(FDBFuture* f) : Future(f) {}
 };

@ -164,6 +165,7 @@ class EmptyFuture : public Future {
 private:
 	friend class Transaction;
 	friend class Database;
+	friend class Tenant;
 	EmptyFuture(FDBFuture* f) : Future(f) {}
 };

@ -221,6 +223,14 @@ public:
 	Tenant(Tenant&&) = delete;
 	Tenant& operator=(Tenant&&) = delete;

+	static KeyFuture purge_blob_granules(FDBTenant* tenant,
+	                                     std::string_view begin_key,
+	                                     std::string_view end_key,
+	                                     int64_t purge_version,
+	                                     fdb_bool_t force);
+
+	static EmptyFuture wait_purge_granules_complete(FDBTenant* tenant, std::string_view purge_key);
+
 private:
 	friend class Transaction;
 	FDBTenant* tenant;
--- a/bindings/c/test/unit/unit_tests.cpp
+++ b/bindings/c/test/unit/unit_tests.cpp
@ -2249,7 +2249,7 @@ TEST_CASE("special-key-space set transaction ID after write") {
 		fdb_check(f1.get(&out_present, (const uint8_t**)&val, &vallen));

 		REQUIRE(out_present);
-		UID transaction_id = UID::fromString(val);
+		UID transaction_id = UID::fromString(std::string(val, vallen));
 		CHECK(transaction_id.first() > 0);
 		CHECK(transaction_id.second() > 0);
 		break;
@ -2316,7 +2316,7 @@ TEST_CASE("special-key-space tracing get range") {
 		CHECK(out_count == 2);

 		CHECK(std::string((char*)out_kv[1].key, out_kv[1].key_length) == tracingBegin + "transaction_id");
-		UID transaction_id = UID::fromString(std::string((char*)out_kv[1].value));
+		UID transaction_id = UID::fromString(std::string((char*)out_kv[1].value, out_kv[1].value_length));
 		CHECK(transaction_id.first() > 0);
 		CHECK(transaction_id.second() > 0);
 		break;
--- a/bindings/flow/CMakeLists.txt
+++ b/bindings/flow/CMakeLists.txt
@ -20,6 +20,7 @@ add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS})
 target_link_libraries(fdb_flow PUBLIC fdb_c)
 target_link_libraries(fdb_flow PUBLIC fdbclient)
 target_include_directories(fdb_flow PUBLIC
+  "${CMAKE_SOURCE_DIR}"
  "${CMAKE_CURRENT_BINARY_DIR}"
  "${CMAKE_CURRENT_SOURCE_DIR}"
  "${CMAKE_CURRENT_SOURCE_DIR}/tester"
--- a/bindings/flow/fdb_flow.actor.cpp
+++ b/bindings/flow/fdb_flow.actor.cpp
@ -24,7 +24,7 @@
 #include <stdio.h>
 #include <cinttypes>

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/SystemMonitor.h"
 #include "flow/TLSConfig.actor.h"
--- a/bindings/flow/tester/Tester.actor.cpp
+++ b/bindings/flow/tester/Tester.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "Tester.actor.h"
+#include "tester/Tester.actor.h"
 #include <cinttypes>
 #ifdef __linux__
 #include <string.h>
--- a/bindings/flow/tester/Tester.actor.h
+++ b/bindings/flow/tester/Tester.actor.h
@ -22,7 +22,7 @@
 // version.
 #if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
 #define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
-#include "Tester.actor.g.h"
+#include "tester/Tester.actor.g.h"
 #elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
 #define FDB_FLOW_TESTER_TESTER_ACTOR_H

--- a/bindings/python/tests/fdbcli_tests.py
+++ b/bindings/python/tests/fdbcli_tests.py
@ -621,6 +621,15 @@ def tenants(logger):
    assert lines[1].strip().startswith('prefix: ')
    assert lines[2].strip().startswith('tenant state: ready')

+    output = run_fdbcli_command('gettenant tenant JSON')
+    json_output = json.loads(output, strict=False)
+    assert(len(json_output) == 2)
+    assert('tenant' in json_output)
+    assert(json_output['type'] == 'success')
+    assert(len(json_output['tenant']) == 2)
+    assert('id' in json_output['tenant'])
+    assert('prefix' in json_output['tenant'])
+
    output = run_fdbcli_command('usetenant')
    assert output == 'Using the default tenant'

--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@ -64,9 +64,6 @@ add_compile_definitions(BOOST_ERROR_CODE_HEADER_ONLY BOOST_SYSTEM_NO_DEPRECATED)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)

-include_directories(${CMAKE_SOURCE_DIR})
-include_directories(${CMAKE_BINARY_DIR})
-
 if(WIN32)
  add_definitions(-DBOOST_USE_WINDOWS_H)
  add_definitions(-DWIN32_LEAN_AND_MEAN)
--- a/cmake/FlowCommands.cmake
+++ b/cmake/FlowCommands.cmake
@ -64,9 +64,8 @@ function(generate_coverage_xml)
      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
      COMMENT "Generate coverage xml")
  endif()
-  add_custom_target(coverage_${target_name} DEPENDS ${target_file})
+  add_custom_target(coverage_${target_name} ALL DEPENDS ${target_file})
  add_dependencies(coverage_${target_name} coveragetool)
-  add_dependencies(${target_name} coverage_${target_name})
 endfunction()

 # This function asserts that `versions.h` does not exist in the source
@ -147,6 +146,47 @@ function(strip_debug_symbols target)
  add_dependencies(strip_targets strip_${target})
 endfunction()

+# This will copy the header from a flow target into ${CMAKE_BINARY_DIR}/include/target-name
+# We're doing this to enforce proper dependencies. In the past we simply added the source
+# and binary dir to the include list, which means that for example a compilation unit in
+# flow could include a header file that lives in fdbserver. This is a somewhat hacky solution
+# but due to our directory structure it seems to be the least invasive one.
+function(copy_headers)
+  set(options)
+  set(oneValueArgs NAME OUT_DIR INC_DIR)
+  set(multiValueArgs SRCS)
+  cmake_parse_arguments(CP "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
+  get_filename_component(dir_name ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+  set(include_dir "${CMAKE_CURRENT_BINARY_DIR}/include")
+  set(incl_dir "${include_dir}/${dir_name}")
+  make_directory("${incl_dir}")
+  foreach(f IN LISTS CP_SRCS)
+    is_prefix(bd "${CMAKE_CURRENT_BINARY_DIR}" "${f}")
+    is_prefix(sd "${CMAKE_CURRENT_SOURCE_DIR}" "${f}")
+    if (bd OR sd)
+      continue()
+    endif()
+    is_header(hdr "${f}")
+    if(NOT hdr)
+      continue()
+    endif()
+    get_filename_component(fname ${f} NAME)
+    get_filename_component(dname ${f} DIRECTORY)
+    if (dname)
+      make_directory(${incl_dir}/${dname})
+    endif()
+    set(fpath "${incl_dir}/${dname}/${fname}")
+    add_custom_command(OUTPUT "${fpath}"
+      DEPENDS "${f}"
+      COMMAND "${CMAKE_COMMAND}" -E copy "${f}" "${fpath}"
+      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+    list(APPEND out_files "${fpath}")
+  endforeach()
+  add_custom_target("${CP_NAME}_incl" DEPENDS ${out_files})
+  set("${CP_OUT_DIR}" "${incl_dir}" PARENT_SCOPE)
+  set("${CP_INC_DIR}" ${include_dir} PARENT_SCOPE)
+endfunction()
+
 function(add_flow_target)
  set(options EXECUTABLE STATIC_LIBRARY
    DYNAMIC_LIBRARY)
@ -159,42 +199,64 @@ function(add_flow_target)
  if(NOT AFT_SRCS)
    message(FATAL_ERROR "No sources provided")
  endif()
+  #foreach(src IN LISTS AFT_SRCS)
+  #  is_header(h "${src}")
+  #  if(NOT h)
+  #    list(SRCS "${CMAKE_CURRENT_SOURCE_DIR}/${src}")
+  #  endif()
+  #endforeach()
  if(OPEN_FOR_IDE)
    # Intentionally omit ${AFT_DISABLE_ACTOR_DIAGNOSTICS} since we don't want diagnostics
    set(sources ${AFT_SRCS} ${AFT_ADDL_SRCS})
    add_library(${AFT_NAME} OBJECT ${sources})
  else()
+    create_build_dirs(${AFT_SRCS} ${AFT_DISABLE_ACTOR_DIAGNOSTICS})
    foreach(src IN LISTS AFT_SRCS AFT_DISABLE_ACTOR_DIAGNOSTICS)
-      set(actor_compiler_flags "")
+      is_header(hdr ${src})
+      set(in_filename "${src}")
      if(${src} MATCHES ".*\\.actor\\.(h|cpp)")
-        list(APPEND actors ${src})
-        list(APPEND actor_compiler_flags "--generate-probes")
+        set(is_actor_file YES)
        if(${src} MATCHES ".*\\.h")
-          string(REPLACE ".actor.h" ".actor.g.h" generated ${src})
+          string(REPLACE ".actor.h" ".actor.g.h" out_filename ${in_filename})
        else()
-          string(REPLACE ".actor.cpp" ".actor.g.cpp" generated ${src})
+          string(REPLACE ".actor.cpp" ".actor.g.cpp" out_filename ${in_filename})
        endif()
+      else()
+        set(is_actor_file NO)
+        set(out_filename "${src}")
+      endif()
+
+      set(in_file "${CMAKE_CURRENT_SOURCE_DIR}/${in_filename}")
+      if(is_actor_file)
+        set(out_file "${CMAKE_CURRENT_BINARY_DIR}/${out_filename}")
+      else()
+        set(out_file "${in_file}")
+      endif()
+
+      list(APPEND sources ${out_file})
+      set(actor_compiler_flags "")
+      if(is_actor_file)
+        list(APPEND actors ${in_file})
+        list(APPEND actor_compiler_flags "--generate-probes")
        foreach(s IN LISTS AFT_DISABLE_ACTOR_DIAGNOSTICS)
          if("${s}" STREQUAL "${src}")
            list(APPEND actor_compiler_flags "--disable-diagnostics")
            break()
          endif()
        endforeach()
-        list(APPEND sources ${generated})
-        list(APPEND generated_files ${CMAKE_CURRENT_BINARY_DIR}/${generated})
+
+        list(APPEND generated_files ${out_file})
        if(WIN32)
-          add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
-            COMMAND $<TARGET_FILE:actorcompiler> "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags}
-            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
+          add_custom_command(OUTPUT "${out_file}"
+            COMMAND $<TARGET_FILE:actorcompiler> "${in_file}" "${out_file}" ${actor_compiler_flags}
+            DEPENDS "${in_file}" ${actor_exe}
            COMMENT "Compile actor: ${src}")
        else()
-          add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
-            COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} > /dev/null
-            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
+          add_custom_command(OUTPUT "${out_file}"
+            COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${in_file}" "${out_file}" ${actor_compiler_flags} > /dev/null
+            DEPENDS "${in_file}" ${actor_exe}
            COMMENT "Compile actor: ${src}")
        endif()
-      else()
-        list(APPEND sources ${src})
      endif()
    endforeach()
    if(AFT_EXECUTABLE)
@ -227,7 +289,6 @@ function(add_flow_target)
    set_property(TARGET ${AFT_NAME} PROPERTY COVERAGE_FILTERS ${AFT_SRCS})

    add_custom_target(${AFT_NAME}_actors DEPENDS ${generated_files})
-    add_dependencies(${AFT_NAME}_actors actorcompiler)
    add_dependencies(${AFT_NAME} ${AFT_NAME}_actors)
    if(NOT WIN32)
      assert_no_version_h(${AFT_NAME}_actors)
--- a/cmake/Jemalloc.cmake
+++ b/cmake/Jemalloc.cmake
@ -9,12 +9,11 @@ add_library(im_jemalloc STATIC IMPORTED)
 include(ExternalProject)
 set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
 ExternalProject_add(Jemalloc_project
-  URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
-  URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
+  URL "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
+  URL_HASH SHA256=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa
  BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
  "${JEMALLOC_DIR}/lib/libjemalloc.a"
  "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
-  PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/cmake/jemalloc.patch
  CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
  BUILD_IN_SOURCE ON
  BUILD_COMMAND make
--- a/cmake/jemalloc.patch
+++ b/cmake/jemalloc.patch
@ -1,38 +0,0 @@
-diff --git a/include/jemalloc/internal/sz.h b/include/jemalloc/internal/sz.h
-index 68e558ab..87bb2280 100644
--- a/include/jemalloc/internal/sz.h
-+++ b/include/jemalloc/internal/sz.h
-@@ -266,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) {
- 	assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
- 
- 	/* Try for a small size class. */
-	if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) {
-+	if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
- 		/*
- 		 * Round size up to the nearest multiple of alignment.
- 		 *
-diff --git a/src/arena.c b/src/arena.c
-index ba50e410..dc7646e6 100644
--- a/src/arena.c
-+++ b/src/arena.c
-@@ -1533,10 +1533,17 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-     bool zero, tcache_t *tcache) {
- 	void *ret;
- 
-	if (usize <= SC_SMALL_MAXCLASS
-	    && (alignment < PAGE
-	    || (alignment == PAGE && (usize & PAGE_MASK) == 0))) {
-+	if (usize <= SC_SMALL_MAXCLASS) {
- 		/* Small; alignment doesn't require special slab placement. */
-+
-+		/* usize should be a result of sz_sa2u() */
-+		assert((usize & (alignment - 1)) == 0);
-+
-+		/*
-+		 * Small usize can't come from an alignment larger than a page.
-+		 */
-+		assert(alignment <= PAGE);
-+
- 		ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize),
- 		    zero, tcache, true);
- 	} else {
--- a/cmake/utils.cmake
+++ b/cmake/utils.cmake
@ -0,0 +1,68 @@
+# sets out_var to YES if filename has extension .h or .hpp, NO otherwise
+function(is_header out_var filename)
+  set(res "NO")
+  get_filename_component(ext "${filename}" LAST_EXT)
+  if((ext STREQUAL ".h") OR (ext STREQUAL ".hpp"))
+    set(res "YES")
+  endif()
+  set("${out_var}" "${res}" PARENT_SCOPE)
+endfunction()
+
+function(remove_prefix out prefix str)
+  string(LENGTH "${prefix}" len)
+  string(SUBSTRING "${str}" ${len} -1 res)
+  set("${out}" "${res}" PARENT_SCOPE)
+endfunction()
+
+function(is_prefix out prefix str)
+  string(LENGTH "${prefix}" plen)
+  string(LENGTH "${str}" slen)
+  if(plen GREATER slen)
+    set(res NO)
+  else()
+    string(SUBSTRING "${str}" 0 ${plen} pstr)
+    if(pstr STREQUAL prefix)
+      set(res YES)
+    else()
+      set(res NO)
+    endif()
+  endif()
+  set(${out} ${res} PARENT_SCOPE)
+endfunction()
+
+function(create_build_dirs)
+  foreach(src IN LISTS ARGV)
+    get_filename_component(d "${src}" DIRECTORY)
+    if(IS_ABSOLUTE "${d}")
+      file(RELATIVE_PATH d "${CMAKE_CURRENT_SOURCE_DIR}" "${src}")
+    endif()
+    list(APPEND dirs "${d}")
+  endforeach()
+  list(REMOVE_DUPLICATES dirs)
+  foreach(dir IN LISTS dirs)
+    make_directory("${CMAKE_CURRENT_BINARY_DIR}/${dir}")
+  endforeach()
+endfunction()
+
+function(fdb_find_sources out)
+  file(GLOB res
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+    CONFIGURE_DEPENDS "*.cpp" "*.c" "*.h" "*.hpp")
+  file(GLOB_RECURSE res_includes
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    CONFIGURE_DEPENDS "include/*.cpp" "include/*.c" "include/*.h" "include/*.hpp")
+  file(GLOB_RECURSE res_workloads
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/workloads"
+    CONFIGURE_DEPENDS "workloads/*.cpp" "workloads/*.c" "workloads/*.h" "workloads/*.hpp")
+
+  foreach(f IN LISTS res_includes)
+    list(APPEND res "include/${f}")
+  endforeach()
+  foreach(f IN LISTS res_workloads)
+    list(APPEND res "workloads/${f}")
+  endforeach()
+  set(${out} "${res}" PARENT_SCOPE)
+endfunction()
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@ -1,8 +1,14 @@
 add_library(rapidjson INTERFACE)
 target_include_directories(rapidjson INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/rapidjson)

+add_subdirectory(crc32)
+add_subdirectory(stacktrace)
+add_subdirectory(folly_memcpy)
+add_subdirectory(sqlite)
+add_subdirectory(SimpleOpt)
 add_subdirectory(fmt-8.1.1)
 if(NOT WIN32)
+  add_subdirectory(linenoise)
  add_subdirectory(debug_determinism)
  add_subdirectory(monitoring)
  add_subdirectory(TraceLogHelper)
--- a/contrib/SimpleOpt/CMakeLists.txt
+++ b/contrib/SimpleOpt/CMakeLists.txt
@ -0,0 +1,2 @@
+add_library(SimpleOpt INTERFACE)
+target_include_directories(SimpleOpt INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")
--- a/contrib/SimpleOpt/include/SimpleOpt/SimpleOpt.h
+++ b/contrib/SimpleOpt/include/SimpleOpt/SimpleOpt.h
@ -78,7 +78,7 @@
    <li> Include the SimpleOpt.h header file

        <pre>
-        \#include "flow/SimpleOpt.h"
+        \#include "SimpleOpt/SimpleOpt.h"
        </pre>

    <li> Define an array of valid options for your program.
--- a/contrib/crc32/CMakeLists.txt
+++ b/contrib/crc32/CMakeLists.txt
@ -0,0 +1,2 @@
+add_library(crc32 STATIC crc32.S crc32_wrapper.c crc32c.cpp)
+target_include_directories(crc32 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
--- a/contrib/crc32/crc32.S
+++ b/contrib/crc32/crc32.S
@ -58,7 +58,7 @@
 #ifdef CRC32_CONSTANTS_HEADER
 #include CRC32_CONSTANTS_HEADER
 #else
-#include "crc32_constants.h"
+#include "crc32/crc32_constants.h"
 #endif

 	.text
--- a/contrib/crc32/crc32_wrapper.c
+++ b/contrib/crc32/crc32_wrapper.c
@ -15,7 +15,7 @@
 #ifdef CRC32_CONSTANTS_HEADER
 #include CRC32_CONSTANTS_HEADER
 #else
-#include "crc32_constants.h"
+#include "crc32/crc32_constants.h"
 #endif

 #define VMX_ALIGN 16
--- a/contrib/crc32/crc32c-generated-constants.cpp
+++ b/contrib/crc32/crc32c-generated-constants.cpp
--- a/contrib/crc32/crc32c.cpp
+++ b/contrib/crc32/crc32c.cpp
@ -25,7 +25,17 @@
 #define _CRT_SECURE_NO_WARNINGS
 #endif

-#include "flow/crc32c.h"
+#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
+#define __unixish__ 1
+#endif
+
+#ifdef __unixish__
+#if !defined(__aarch64__) && !defined(__powerpc64__)
+#include <cpuid.h>
+#endif
+#endif
+
+#include "crc32/crc32c.h"

 #if !defined(__aarch64__) && !defined(__powerpc64__)
 #include <nmmintrin.h>
@ -34,9 +44,40 @@
 #include <stdlib.h>
 #include <random>
 #include <algorithm>
-#include "flow/Platform.h"
 #include "crc32c-generated-constants.cpp"

+// CRC32C
+#ifdef __aarch64__
+// aarch64
+#include <inttypes.h>
+static inline uint32_t hwCrc32cU8(unsigned int crc, unsigned char v) {
+	uint32_t ret;
+	asm volatile("crc32cb %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+static inline uint32_t hwCrc32cU32(unsigned int crc, unsigned int v) {
+	uint32_t ret;
+	asm volatile("crc32cw %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+#ifdef _M_X64
+static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
+	uint64_t ret;
+	asm volatile("crc32cx %w[r], %w[c], %x[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+#endif
+#else
+#ifndef __powerpc64__
+// Intel
+#define hwCrc32cU8(c, v) _mm_crc32_u8(c, v)
+#define hwCrc32cU32(c, v) _mm_crc32_u32(c, v)
+#ifdef _M_X64
+#define hwCrc32cU64(c, v) _mm_crc32_u64(c, v)
+#endif
+#endif
+#endif
+
 [[maybe_unused]] static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length) {
 	for (size_t i = 0; i < length; ++i) {
 		crc = crc ^ input[i];
@ -278,7 +319,25 @@ uint32_t ppc_hw(uint32_t crc, const uint8_t* input, size_t length) {
 }
 #endif

-static bool hw_available = platform::isHwCrcSupported();
+bool isHwCrcSupported() {
+#if defined(_WIN32)
+	int info[4];
+	__cpuid(info, 1);
+	return (info[2] & (1 << 20)) != 0;
+#elif defined(__aarch64__)
+	return true; /* force to use crc instructions */
+#elif defined(__powerpc64__)
+	return false; /* force not to use crc instructions */
+#elif defined(__unixish__)
+	uint32_t eax, ebx, ecx, edx, level = 1, count = 0;
+	__cpuid_count(level, count, eax, ebx, ecx, edx);
+	return ((ecx >> 20) & 1) != 0;
+#else
+#error Port me!
+#endif
+}
+
+static bool hw_available = isHwCrcSupported();

 extern "C" uint32_t crc32c_append(uint32_t crc, const uint8_t* input, size_t length) {
 	if (hw_available) {
--- a/contrib/crc32/include/crc32/crc32_constants.h
+++ b/contrib/crc32/include/crc32/crc32_constants.h
--- a/contrib/crc32/include/crc32/crc32_wrapper.h
+++ b/contrib/crc32/include/crc32/crc32_wrapper.h
--- a/contrib/crc32/include/crc32/crc32c.h
+++ b/contrib/crc32/include/crc32/crc32c.h
--- a/contrib/folly_memcpy/CMakeLists.txt
+++ b/contrib/folly_memcpy/CMakeLists.txt
@ -0,0 +1,4 @@
+if(UNIX AND NOT APPLE)
+  add_library(folly_memcpy STATIC folly_memcpy.S)
+  target_include_directories(folly_memcpy PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
+endif()
--- a/contrib/folly_memcpy/folly_memcpy.S
+++ b/contrib/folly_memcpy/folly_memcpy.S
--- a/contrib/folly_memcpy/folly_memcpy.h
+++ b/contrib/folly_memcpy/folly_memcpy.h
--- a/contrib/linenoise/CMakeLists.txt
+++ b/contrib/linenoise/CMakeLists.txt
@ -0,0 +1,2 @@
+add_library(linenoise STATIC linenoise.c)
+target_include_directories(linenoise PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
--- a/contrib/linenoise/include/linenoise/linenoise.h
+++ b/contrib/linenoise/include/linenoise/linenoise.h
--- a/contrib/linenoise/linenoise.c
+++ b/contrib/linenoise/linenoise.c
@ -115,7 +115,7 @@
 #include <sys/types.h>
 #include <sys/ioctl.h>
 #include <unistd.h>
-#include "linenoise.h"
+#include "linenoise/linenoise.h"

 #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
 #define LINENOISE_MAX_LINE 4096
--- a/contrib/mockkms/mock_kms.go
+++ b/contrib/mockkms/mock_kms.go
@ -24,6 +24,15 @@

 package main

+// Importing module enable .note.gnu.build-id insertion in the ELF executable.
+// Few drawbacks of the scheme are:
+// 1. Potentialy slower builds
+// 2. No cross-compilation support
+// 3. Limited `go tools` availability.
+// TODO: Replace with a better scheme if possible.
+
+import "C"
+
 import (
    "log"
    "math/rand"
--- a/contrib/sqlite/CMakeLists.txt
+++ b/contrib/sqlite/CMakeLists.txt
@ -0,0 +1,16 @@
+add_library(sqlite STATIC
+  btree.h
+  hash.h
+  sqlite3.h
+  sqlite3ext.h
+  sqliteInt.h
+  sqliteLimit.h
+  sqlite3.amalgamation.c)
+
+
+target_include_directories(sqlite PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+# Suppress warnings in sqlite since it's third party
+if(NOT WIN32)
+  target_compile_definitions(sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
+  target_compile_options(sqlite BEFORE PRIVATE -w) # disable warnings for third party
+endif()
--- a/fdbserver/sqlite/btree.c
+++ b/fdbserver/sqlite/btree.c
--- a/fdbserver/sqlite/btree.h
+++ b/fdbserver/sqlite/btree.h
--- a/fdbserver/sqlite/hash.h
+++ b/fdbserver/sqlite/hash.h
--- a/fdbserver/sqlite/shell.c
+++ b/fdbserver/sqlite/shell.c
--- a/fdbserver/sqlite/sqlite3.amalgamation.c
+++ b/fdbserver/sqlite/sqlite3.amalgamation.c
--- a/fdbserver/sqlite/sqlite3.h
+++ b/fdbserver/sqlite/sqlite3.h
--- a/fdbserver/sqlite/sqlite3ext.h
+++ b/fdbserver/sqlite/sqlite3ext.h
--- a/fdbserver/sqlite/sqliteInt.h
+++ b/fdbserver/sqlite/sqliteInt.h
--- a/fdbserver/sqlite/sqliteLimit.h
+++ b/fdbserver/sqlite/sqliteLimit.h
--- a/contrib/stacktrace/CMakeLists.txt
+++ b/contrib/stacktrace/CMakeLists.txt
@ -0,0 +1,11 @@
+add_library(stacktrace STATIC stacktrace.amalgamation.cpp)
+target_include_directories(stacktrace PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+if (USE_ASAN)
+  target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER)
+elseif(USE_MSAN)
+  target_compile_definitions(stacktrace PRIVATE MEMORY_SANITIZER)
+elseif(USE_UBSAN)
+  target_compile_definitions(stacktrace PRIVATE UNDEFINED_BEHAVIOR_SANITIZER)
+elseif(USE_TSAN)
+  target_compile_definitions(stacktrace PRIVATE THREAD_SANITIZER DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1)
+endif()
--- a/contrib/stacktrace/include/stacktrace/stacktrace.h
+++ b/contrib/stacktrace/include/stacktrace/stacktrace.h
--- a/contrib/stacktrace/stacktrace.amalgamation.cpp
+++ b/contrib/stacktrace/stacktrace.amalgamation.cpp
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_aarch64-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_aarch64-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_arm-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_arm-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_generic-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_generic-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_powerpc-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_powerpc-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_unimplemented-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_unimplemented-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_win32-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_win32-inl.inc
--- a/contrib/stacktrace/stacktrace_internal/stacktrace_x86-inl.inc
+++ b/contrib/stacktrace/stacktrace_internal/stacktrace_x86-inl.inc
--- a/documentation/sphinx/source/administration.rst
+++ b/documentation/sphinx/source/administration.rst
@ -762,6 +762,11 @@ You can now remove old client library versions from your clients. This is only t
 Version-specific notes on upgrading
 ===================================

+Upgrading to 7.1.x
+--------------------
+
+Upgrades to 7.1.0 or later will break any client using ``fdb_transaction_get_range_and_flat_map``, as it is removed in version 7.1.0.
+
 Upgrading from 6.2.x
 --------------------

--- a/documentation/sphinx/source/api-version-upgrade-guide.rst
+++ b/documentation/sphinx/source/api-version-upgrade-guide.rst
@ -27,6 +27,8 @@ API version 710
 General
 -------

+* ``fdb_transaction_get_range_and_flat_map`` API is replaced by ``fdb_transaction_get_mapped_range`` in version 710. The function ``fdb_transaction_get_range_and_flat_map`` is not supported in any API version.
+
 .. _api-version-upgrade-guide-700:

 API version 700
--- a/documentation/sphinx/source/backups.rst
+++ b/documentation/sphinx/source/backups.rst
@ -83,7 +83,7 @@ For blob store backup locations, the Backup URL format is

 ::

-    blobstore://[<api_key>][:<secret>[:<security_token>]]@<hostname>[:<port>]/<name>?bucket=<bucket_name>[&<param>=<value>]...]
+    blobstore://[<api_key>][:<secret>[:<security_token>]]@<hostname>[:<port>]/<name>?bucket=<bucket_name>[&region=<region_name>][&<param>=<value>]...]

      <api_key> - API key to use for authentication. Optional.
      <secret> - API key's secret.  Optional.
@ -92,6 +92,7 @@ For blob store backup locations, the Backup URL format is
      <port> - Remote port to connect to.  Optional.  Default is 80.
      <name> - Name of the backup within the backup bucket.  It can contain '/' characters in order to organize backups into a folder-like structure.
      <bucket_name> - Name of the bucket to use for backup data.
+      <region_name> - If <hostname> is not in s3 compatible form (s3.region-name.example.com) and aws v4 signature is enabled, region name is required.
      
      <param>=<value> - Optional URL parameters.  See below for details.

--- a/documentation/sphinx/source/command-line-interface.rst
+++ b/documentation/sphinx/source/command-line-interface.rst
@ -234,12 +234,27 @@ Note that :ref:`characters can be escaped <cli-escaping>` when specifying keys (
 gettenant
 ---------

-The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME>``.
+The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME> [JSON]``.

-Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error.
+Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error. If ``JSON`` is specified, then the output will be written as a JSON document::

-getversion
----------
+    {
+        "tenant": {
+            "id": 0,
+            "prefix": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
+        },
+        "type": "success"
+    }
+
+In the event of an error, the output will include an error message::
+
+    {
+        "error": "...",
+        "type": "error"
+    }
+
+    getversion
+    ----------

 The ``getversion`` command fetches the current read version of the cluster or currently running transaction.

--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@ -4,6 +4,43 @@
 Release Notes
 #############

+7.1.11
+======
+* Same as 7.1.10 release with AVX enabled.
+
+7.1.10
+======
+* Released with AVX disabled.
+* Fixed a sequencer crash when DC ID is a string. `(PR #7393) <https://github.com/apple/foundationdb/pull/7393>`_
+* Fixed a client performance regression by removing unnecessary transaction initialization. `(PR #7365) <https://github.com/apple/foundationdb/pull/7365>`_
+* Safely removed fdb_transaction_get_range_and_flat_map C API. `(PR #7379) <https://github.com/apple/foundationdb/pull/7379>`_
+* Fixed an unknown error bug when hostname resolving fails. `(PR #7380) <https://github.com/apple/foundationdb/pull/7380>`_
+
+7.1.9
+=====
+* Same as 7.1.8 release with AVX enabled.
+
+7.1.8
+=====
+* Released with AVX disabled.
+* Fixed a performance regression in network run loop.  `(PR #7342) <https://github.com/apple/foundationdb/pull/7342>`_
+* Added RSS bytes for processes in status json output and corrected available_bytes calculation. `(PR #7348) <https://github.com/apple/foundationdb/pull/7348>`_
+* Added versionstamp support in tuples. `(PR #7313) <https://github.com/apple/foundationdb/pull/7313>`_
+* Fixed some spammy trace events. `(PR #7300) <https://github.com/apple/foundationdb/pull/7300>`_
+* Avoided a memory corruption bug by disabling streaming peeks. `(PR #7288) <https://github.com/apple/foundationdb/pull/7288>`_
+* Fixed a hang bug in fdbcli exclude command. `(PR #7268) <https://github.com/apple/foundationdb/pull/7268>`_
+* Fixed an issue that a remote TLog blocks peeks. `(PR #7255) <https://github.com/apple/foundationdb/pull/7255>`_
+* Fixed a connection issue using hostnames. `(PR #7264) <https://github.com/apple/foundationdb/pull/7264>`_
+* Added support of the reboot command in go bindings. `(PR #7270) <https://github.com/apple/foundationdb/pull/7270>`_
+* Fixed several issues in profiling special keys using GlobalConfig. `(PR #7120) <https://github.com/apple/foundationdb/pull/7120>`_
+* Fixed a stuck transaction system bug due to inconsistent recovery transaction version. `(PR #7261) <https://github.com/apple/foundationdb/pull/7261>`_
+* Fixed a unknown_error crash due to not resolving hostnames. `(PR #7254) <https://github.com/apple/foundationdb/pull/7254>`_
+* Fixed a heap-use-after-free bug. `(PR #7250) <https://github.com/apple/foundationdb/pull/7250>`_
+* Fixed a performance issue that remote TLogs are sending too many pops to log routers. `(PR #7235) <https://github.com/apple/foundationdb/pull/7235>`_
+* Fixed an issue that SharedTLogs are not displaced and leaking disk space. `(PR #7246) <https://github.com/apple/foundationdb/pull/7246>`_
+* Fixed an issue that coordinatorsKey does not store DNS names. `(PR #7203) <https://github.com/apple/foundationdb/pull/7203>`_
+* Fixed a sequential execution issue for fdbcli kill, suspend, and expensive_data_check commands. `(PR #7211) <https://github.com/apple/foundationdb/pull/7211>`_
+
 7.1.7
 =====
 * Same as 7.1.6 release with AVX enabled.
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@ -19,7 +19,7 @@
 * limitations under the License.
 */

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "flow/flow.h"
 #include "flow/Platform.h"
 #include "flow/DeterministicRandom.h"
--- a/fdbbackup/CMakeLists.txt
+++ b/fdbbackup/CMakeLists.txt
@ -1,23 +1,22 @@
 set(FDBBACKUP_SRCS
-	BackupTLSConfig.h
 	BackupTLSConfig.cpp
 	backup.actor.cpp)

 add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS})
+target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbbackup PRIVATE fdbclient)

 set(FDBCONVERT_SRCS
-	FileConverter.actor.cpp
-	FileConverter.h)
+	FileConverter.actor.cpp)
 add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS})
+target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbconvert PRIVATE fdbclient)

 set(FDBDECODE_SRCS
-	BackupTLSConfig.h
 	BackupTLSConfig.cpp
-	FileDecoder.actor.cpp
-	FileConverter.h)
+	FileDecoder.actor.cpp)
 add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS})
+target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbdecode PRIVATE fdbclient)

 if(NOT OPEN_FOR_IDE)
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbbackup/BackupTLSConfig.h"
 #include "fdbclient/JsonBuilder.h"
 #include "flow/Arena.h"
@ -74,7 +74,7 @@
 #include "fdbclient/versions.h"
 #include "fdbclient/BuildFlags.h"

-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "flow/actorcompiler.h" // This must be the last #include.

 // Type of program being executed
--- a/fdbbackup/include/fdbbackup/BackupTLSConfig.h
+++ b/fdbbackup/include/fdbbackup/BackupTLSConfig.h
--- a/fdbbackup/include/fdbbackup/FileConverter.h
+++ b/fdbbackup/include/fdbbackup/FileConverter.h
@ -23,7 +23,7 @@
 #pragma once

 #include <cinttypes>
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "flow/TLSConfig.actor.h"

 namespace file_converter {
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@ -19,7 +19,7 @@
 */

 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbcli/fdbcli.actor.h"

 #include "fdbclient/IClientApi.h"
--- a/fdbcli/CMakeLists.txt
+++ b/fdbcli/CMakeLists.txt
@ -1,44 +1,12 @@
-set(FDBCLI_SRCS
-  fdbcli.actor.cpp
-  fdbcli.actor.h
-  AdvanceVersionCommand.actor.cpp
-  BlobRangeCommand.actor.cpp
-  CacheRangeCommand.actor.cpp
-  ConfigureCommand.actor.cpp
-  ConsistencyCheckCommand.actor.cpp
-  CoordinatorsCommand.actor.cpp
-  DataDistributionCommand.actor.cpp
-  ExcludeCommand.actor.cpp
-  ExpensiveDataCheckCommand.actor.cpp
-  FileConfigureCommand.actor.cpp
-  FlowLineNoise.actor.cpp
-  FlowLineNoise.h
-  ForceRecoveryWithDataLossCommand.actor.cpp
-  IncludeCommand.actor.cpp
-  KillCommand.actor.cpp
-  LockCommand.actor.cpp
-  ChangeFeedCommand.actor.cpp
-  MaintenanceCommand.actor.cpp
-  MetaclusterCommands.actor.cpp
-  ProfileCommand.actor.cpp
-  SetClassCommand.actor.cpp
-  SnapshotCommand.actor.cpp
-  StatusCommand.actor.cpp
-  SuspendCommand.actor.cpp
-  TenantCommands.actor.cpp
-  ThrottleCommand.actor.cpp
-  TriggerDDTeamInfoLogCommand.actor.cpp
-  TssqCommand.actor.cpp
-  Util.actor.cpp
-  VersionEpochCommand.actor.cpp
-  linenoise/linenoise.h)
-
-if(NOT WIN32)
-  list(APPEND FDBCLI_SRCS linenoise/linenoise.c)
-endif()
+fdb_find_sources(FDBCLI_SRCS)

 add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
-target_link_libraries(fdbcli PRIVATE fdbclient)
+target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_link_libraries(fdbcli PRIVATE fdbclient SimpleOpt)
+
+if(NOT WIN32)
+  target_link_libraries(fdbcli PRIVATE linenoise)
+endif()

 if(NOT OPEN_FOR_IDE)
  if(GENERATE_DEBUG_PACKAGES)
--- a/fdbcli/ChangeFeedCommand.actor.cpp
+++ b/fdbcli/ChangeFeedCommand.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"

 #include "fdbcli/fdbcli.actor.h"

--- a/fdbcli/FlowLineNoise.actor.cpp
+++ b/fdbcli/FlowLineNoise.actor.cpp
@ -30,7 +30,7 @@

 #if __unixish__
 #define HAVE_LINENOISE 1
-#include "fdbcli/linenoise/linenoise.h"
+#include "linenoise/linenoise.h"
 #else
 #define HAVE_LINENOISE 0
 #endif
--- a/fdbcli/MaintenanceCommand.actor.cpp
+++ b/fdbcli/MaintenanceCommand.actor.cpp
@ -21,7 +21,7 @@
 #include <cinttypes>

 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"

 #include "fdbcli/fdbcli.actor.h"

--- a/fdbcli/SetClassCommand.actor.cpp
+++ b/fdbcli/SetClassCommand.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"

 #include "fdbcli/fdbcli.actor.h"

--- a/fdbcli/TenantCommands.actor.cpp
+++ b/fdbcli/TenantCommands.actor.cpp
@ -301,7 +301,6 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
 				printf("%s\n",
 				       json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
 			} else {
-
 				JSONDoc doc(jsonObject);

 				int64_t id;
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@ -19,7 +19,7 @@
 */

 #include "boost/lexical_cast.hpp"
-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/FDBTypes.h"
@ -39,6 +39,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbclient/Tuple.h"

 #include "fdbclient/ThreadSafeTransaction.h"
@ -51,7 +52,7 @@

 #include "flow/TLSConfig.actor.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"

 #include "fdbcli/FlowLineNoise.h"
 #include "fdbcli/fdbcli.actor.h"
@ -63,7 +64,7 @@

 #ifdef __unixish__
 #include <stdio.h>
-#include "fdbcli/linenoise/linenoise.h"
+#include "linenoise/linenoise.h"
 #endif

 #include "fdbclient/versions.h"
@ -1049,7 +1050,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	state Database localDb;
 	state Reference<IDatabase> db;
 	state Reference<ITenant> tenant;
-	state Optional<Standalone<StringRef>> tenantName;
+	state Optional<TenantName> tenantName;
 	state Optional<TenantMapEntry> tenantEntry;

 	// This tenant is kept empty for operations that perform management tasks (e.g. killing a process)
--- a/fdbcli/include/fdbcli/FlowLineNoise.h
+++ b/fdbcli/include/fdbcli/FlowLineNoise.h
--- a/fdbcli/include/fdbcli/fdbcli.actor.h
+++ b/fdbcli/include/fdbcli/fdbcli.actor.h
--- a/fdbclient/ActorLineageProfiler.cpp
+++ b/fdbclient/ActorLineageProfiler.cpp
@ -20,7 +20,7 @@

 #include "flow/flow.h"
 #include "flow/singleton.h"
-#include "fdbrpc/IAsyncFile.h"
+#include "flow/IAsyncFile.h"
 #include "fdbclient/ActorLineageProfiler.h"
 #include "fdbclient/NameLineage.h"
 #include <msgpack.hpp>
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@ -22,7 +22,7 @@
 #include <ostream>

 // FIXME: Trim this down
-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "flow/Platform.actor.h"
 #include "fdbclient/AsyncTaskThread.h"
 #include "fdbclient/BackupContainer.h"
@ -37,7 +37,9 @@
 #include "fdbrpc/simulator.h"
 #include "flow/Platform.h"
 #include "fdbclient/AsyncFileS3BlobStore.actor.h"
+#ifdef BUILD_AZURE_BACKUP
 #include "fdbclient/BackupContainerAzureBlobStore.h"
+#endif
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BackupContainerLocalDirectory.h"
 #include "fdbclient/BackupContainerS3BlobStore.h"
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@ -19,7 +19,9 @@
 */

 #include "fdbclient/BackupAgent.actor.h"
+#ifdef BUILD_AZURE_BACKUP
 #include "fdbclient/BackupContainerAzureBlobStore.h"
+#endif
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BackupContainerLocalDirectory.h"
 #include "fdbclient/BackupContainerS3BlobStore.h"
--- a/fdbclient/BackupContainerLocalDirectory.actor.cpp
+++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp
@ -20,7 +20,7 @@

 #include "fdbclient/BackupContainerLocalDirectory.h"
 #include "fdbrpc/AsyncFileReadAhead.actor.h"
-#include "fdbrpc/IAsyncFile.h"
+#include "flow/IAsyncFile.h"
 #include "flow/Platform.actor.h"
 #include "flow/Platform.h"
 #include "fdbrpc/simulator.h"
--- a/fdbclient/BlobGranuleFiles.cpp
+++ b/fdbclient/BlobGranuleFiles.cpp
@ -20,7 +20,7 @@

 #include <vector>

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "flow/serialize.h"
 #include "fdbclient/BlobGranuleFiles.h"
 #include "fdbclient/Knobs.h"
--- a/fdbclient/BlobGranuleReader.actor.cpp
+++ b/fdbclient/BlobGranuleReader.actor.cpp
@ -21,7 +21,7 @@
 #include <map>
 #include <vector>

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbclient/AsyncFileS3BlobStore.actor.h"
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/BlobGranuleFiles.h"
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@ -1,195 +1,40 @@
-set(FDBCLIENT_SRCS
-  ActorLineageProfiler.h
-  ActorLineageProfiler.cpp
-  AnnotateActor.cpp
-  AsyncFileS3BlobStore.actor.cpp
-  AsyncFileS3BlobStore.actor.h
-  AsyncTaskThread.actor.cpp
-  AsyncTaskThread.h
-  Atomic.h
-  AutoPublicAddress.cpp
-  BackupAgent.actor.h
-  BackupAgentBase.actor.cpp
-  BackupContainer.actor.cpp
-  BackupContainer.h
-  BackupContainerFileSystem.actor.cpp
-  BackupContainerFileSystem.h
-  BackupContainerLocalDirectory.actor.cpp
-  BackupContainerLocalDirectory.h
-  BackupContainerS3BlobStore.actor.cpp
-  BackupContainerS3BlobStore.h
-  BlobConnectionProvider.h
-  BlobConnectionProvider.cpp
-  BlobWorkerInterface.h
-  BlobGranuleReader.actor.cpp
-  BlobGranuleReader.actor.h
-  BlobGranuleCommon.h
-  BlobGranuleFiles.cpp
-  BlobGranuleFiles.h
-  BlobMetadataUtils.h
-  BlobWorkerCommon.h
-  ClientBooleanParams.cpp
-  ClientBooleanParams.h
-  ClientKnobCollection.cpp
-  ClientKnobCollection.h
-  ClientKnobs.cpp
-  ClientKnobs.h
-  ClientLogEvents.h
-  ClientVersion.h
-  ClientWorkerInterface.h
-  ClusterConnectionFile.actor.cpp
-  ClusterConnectionFile.h
-  ClusterConnectionKey.actor.h
-  ClusterConnectionMemoryRecord.actor.cpp
-  ClusterConnectionMemoryRecord.h
-  ClusterInterface.h
-  CommitProxyInterface.h
-  CommitTransaction.h
-  ConfigKnobs.cpp
-  ConfigKnobs.h
-  ConfigTransactionInterface.cpp
-  ConfigTransactionInterface.h
-  ConvertUTF.h
-  CoordinationInterface.h
-  DatabaseBackupAgent.actor.cpp
-  DatabaseConfiguration.cpp
-  DatabaseConfiguration.h
-  DatabaseContext.h
-  EventTypes.actor.h
-  FDBOptions.h
-  FDBTypes.cpp
-  FDBTypes.h
-  FluentDSampleIngestor.cpp
-  FileBackupAgent.actor.cpp
-  GenericManagementAPI.actor.h
-  GenericTransactionHelper.h
-  GlobalConfig.h
-  GlobalConfig.actor.h
-  GlobalConfig.actor.cpp
-  GrvProxyInterface.h
-  HighContentionPrefixAllocator.actor.h
-  IClientApi.h
-  IConfigTransaction.cpp
-  IConfigTransaction.h
-  ISingleThreadTransaction.cpp
-  ISingleThreadTransaction.h
-  JsonBuilder.cpp
-  JsonBuilder.h
-  KeyBackedTypes.h
-  KeyRangeMap.actor.cpp
-  KeyRangeMap.h
-  Knobs.h
-  IKnobCollection.cpp
-  IKnobCollection.h
-  LocalClientAPI.cpp
-  LocalClientAPI.h
-  ManagementAPI.actor.cpp
-  ManagementAPI.actor.h
-  Metacluster.cpp
-  Metacluster.h
-  MetaclusterManagement.actor.cpp
-  MetaclusterManagement.actor.h
-  MonitorLeader.actor.cpp
-  MonitorLeader.h
-  MultiVersionAssignmentVars.h
-  MultiVersionTransaction.actor.cpp
-  MultiVersionTransaction.h
-  MutationList.h
-  MutationLogReader.actor.cpp
-  MutationLogReader.actor.h
-  NameLineage.h
-  NameLineage.cpp
-  NativeAPI.actor.cpp
-  NativeAPI.actor.h
-  Notified.h
-  ParallelStream.actor.cpp
-  ParallelStream.actor.h
-  PaxosConfigTransaction.actor.cpp
-  PaxosConfigTransaction.h
-  PImpl.h
-  SimpleConfigTransaction.actor.cpp
-  SpecialKeySpace.actor.cpp
-  SpecialKeySpace.actor.h
-  ReadYourWrites.actor.cpp
-  ReadYourWrites.h
-  RestoreInterface.cpp
-  RestoreInterface.h
-  RunTransaction.actor.h
-  RYWIterator.cpp
-  RYWIterator.h
-  S3BlobStore.actor.cpp
-  Schemas.cpp
-  Schemas.h
-  ServerKnobCollection.cpp
-  ServerKnobCollection.h
-  ServerKnobs.cpp
-  ServerKnobs.h
-  SimpleConfigTransaction.h
-  SimpleIni.h
-  SnapshotCache.h
-  SpecialKeySpace.actor.cpp
-  SpecialKeySpace.actor.h
-  Status.h
-  StatusClient.actor.cpp
-  StatusClient.h
-  StorageServerInterface.cpp
-  StorageServerInterface.h
-  StorageCheckpoint.h
-  Subspace.cpp
-  Subspace.h
-  StackLineage.h
-  StackLineage.cpp
-  SystemData.cpp
-  SystemData.h
-  TagThrottle.actor.cpp
-  TagThrottle.actor.h
-  TaskBucket.actor.cpp
-  TaskBucket.h
-  Tenant.cpp
-  Tenant.h
-  TenantManagement.actor.cpp
-  TenantManagement.actor.h
-  TenantSpecialKeys.actor.cpp
-  TestKnobCollection.cpp
-  TestKnobCollection.h
-  ThreadSafeTransaction.cpp
-  ThreadSafeTransaction.h
-  Tuple.cpp
-  Tuple.h
-  VersionedMap.actor.h
-  VersionedMap.h
-  VersionedMap.cpp
-  Versionstamp.cpp
-  Versionstamp.h
-  VersionVector.h
-  VersionVector.cpp
-  WellKnownEndpoints.h
-  WriteMap.h
-  WriteMap.cpp
-  json_spirit/json_spirit_error_position.h
-  json_spirit/json_spirit_reader_template.h
-  json_spirit/json_spirit_value.h
-  json_spirit/json_spirit_writer_options.h
-  json_spirit/json_spirit_writer_template.h
-  libb64/cdecode.c
-  libb64/cencode.c
-  md5/md5.c
+fdb_find_sources(FDBCLIENT_SRCS)
+list(APPEND FDBCLIENT_SRCS
  sha1/SHA1.cpp
-  zipf.c
-  zipf.h)
+  libb64/cdecode.c
+  libb64/cencode.c)
+
+message(STATUS "FDB version is ${FDB_VERSION}")
+message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}")

 set(options_srcs ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp)

-vexillographer_compile(TARGET fdboptions LANG cpp OUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g
+make_directory(${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/)
+vexillographer_compile(TARGET fdboptions_vex LANG cpp OUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp)

+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/FDBOptions.g.h
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h
+  COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/FDBOptions.g.h)
+
+vexillographer_compile(TARGET fdboptions_c LANG c OUT ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/fdb_c_options.g.h
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/fdb_c_options.g.h)
+
+add_custom_target(fdboptions DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/FDBOptions.g.h)
+add_dependencies(fdboptions fdboptions_c)
+
+# ###############################################################################
+# Build information
+# ###############################################################################
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/BuildFlags.h)
+
 set(BUILD_AZURE_BACKUP OFF CACHE BOOL "Build Azure backup client")
+
 if(BUILD_AZURE_BACKUP)
  add_compile_definitions(BUILD_AZURE_BACKUP)
  set(FDBCLIENT_SRCS
    ${FDBCLIENT_SRCS}
-    BackupContainerAzureBlobStore.actor.cpp
-    BackupContainerAzureBlobStore.h)
+    azure_backup/BackupContainerAzureBlobStore.actor.cpp)

  configure_file(azurestorage.cmake azurestorage-download/CMakeLists.txt)

@ -219,7 +64,6 @@ if(BUILD_AZURE_BACKUP)
  )
 endif()

-
 if(WITH_AWS_BACKUP)
  add_compile_definitions(BUILD_AWS_BACKUP)

@ -231,6 +75,8 @@ if(WITH_AWS_BACKUP)
 endif()

 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
+target_include_directories(fdbclient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/versions.h)
 add_dependencies(fdbclient fdboptions)
 target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)

@ -238,9 +84,11 @@ target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
 # fdbserver retain sampling functionality in client code while disabling
 # sampling for pure clients.
 add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
+target_include_directories(fdbclient_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 add_dependencies(fdbclient_sampling fdboptions)
 target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack)
 target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)
+
 if(WIN32)
  add_dependencies(fdbclient_sampling_actors fdbclient_actors)
 endif()
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@ -287,6 +287,10 @@ void ClientKnobs::initialize(Randomize randomize) {
 	// Blob granules
 	init( BG_MAX_GRANULE_PARALLELISM,                10 );

+	init( CHANGE_QUORUM_BAD_STATE_RETRY_TIMES,        3 );
+	init( CHANGE_QUORUM_BAD_STATE_RETRY_DELAY,      2.0 );
+
+	// Tenants and Metacluster
 	init( MAX_TENANTS_PER_CLUSTER,                  1e6 );
 	init( MAX_DATA_CLUSTERS,                        1e5 );
 	init( REMOVE_CLUSTER_TENANT_BATCH_SIZE,         1e4 ); if ( randomize && BUGGIFY ) REMOVE_CLUSTER_TENANT_BATCH_SIZE = 1;
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.actor.cpp
@ -26,7 +26,7 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include <ctime>
 #include <climits>
-#include "fdbrpc/IAsyncFile.h"
+#include "flow/IAsyncFile.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Hash3.h"
 #include <numeric>
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/DatabaseContext.h"
@ -33,7 +33,7 @@
 #include <cinttypes>
 #include <ctime>
 #include <climits>
-#include "fdbrpc/IAsyncFile.h"
+#include "flow/IAsyncFile.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Hash3.h"
 #include <numeric>
--- a/fdbclient/GlobalConfig.actor.cpp
+++ b/fdbclient/GlobalConfig.actor.cpp
@ -153,18 +153,6 @@ void GlobalConfig::erase(KeyRangeRef range) {
 	}
 }

-// Similar to tr.onError(), but doesn't require a DatabaseContext.
-struct Backoff {
-	Future<Void> onError() {
-		double currentBackoff = backoff;
-		backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->DEFAULT_MAX_BACKOFF);
-		return delay(currentBackoff * deterministicRandom()->random01());
-	}
-
-private:
-	double backoff = CLIENT_KNOBS->DEFAULT_BACKOFF;
-};
-
 // Older FDB versions used different keys for client profiling data. This
 // function performs a one-time migration of data in these keys to the new
 // global configuration key space.
--- a/fdbclient/HTTP.actor.cpp
+++ b/fdbclient/HTTP.actor.cpp
@ -18,10 +18,12 @@
 * limitations under the License.
 */

-#include "fdbrpc/HTTP.h"
+#include "fdbclient/HTTP.h"

 #include "fdbclient/md5/md5.h"
+#include "fdbclient/ClientKnobs.h"
 #include "fdbclient/libb64/encode.h"
+#include "fdbclient/Knobs.h"
 #include <cctype>

 #include "flow/actorcompiler.h" // has to be last include
--- a/fdbclient/ISingleThreadTransaction.cpp
+++ b/fdbclient/ISingleThreadTransaction.cpp
@ -26,9 +26,14 @@

 ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type type) {
 	if (type == Type::RYW) {
-		auto tr = new ReadYourWritesTransaction;
+		// We only want to allocate memory for the transaction, not initialize
+		// the object, so use operator new instead of new. The RYWTransaction
+		// will get initialized on the main thread.
+		auto tr =
+		    (ReadYourWritesTransaction*)ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction));
 		return tr;
 	} else if (type == Type::SIMPLE_CONFIG) {
+		// Configuration transaction objects expect to be initialized.
 		auto tr = new SimpleConfigTransaction;
 		return tr;
 	} else if (type == Type::PAXOS_CONFIG) {
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@ -22,7 +22,7 @@
 #include <string>
 #include <vector>

-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"
 #include "fdbclient/Knobs.h"
 #include "flow/Arena.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
@ -798,23 +798,60 @@ ACTOR Future<Optional<ClusterConnectionString>> getConnectionString(Database cx)
 	}
 }

-ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
-                                                               ClusterConnectionString* conn,
-                                                               std::string newName) {
+namespace {
+
+ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromStorageServer(Transaction* tr) {
+
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	Optional<Value> currentKey = wait(tr->get(coordinatorsKey));

-	if (!currentKey.present())
-		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
+	state int retryTimes = 0;
+	loop {
+		if (retryTimes >= CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_TIMES) {
+			return Optional<ClusterConnectionString>();
+		}

-	state ClusterConnectionString old(currentKey.get().toString());
-	if (tr->getDatabase()->getConnectionRecord() &&
-	    old.clusterKeyName().toString() !=
-	        tr->getDatabase()->getConnectionRecord()->getConnectionString().clusterKeyName())
-		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
+		Version readVersion = wait(tr->getReadVersion());
+		state Optional<Value> currentKey = wait(tr->get(coordinatorsKey));
+
+		if (!currentKey.present()) {
+			// Someone deleted this key entirely?
+			++retryTimes;
+			wait(delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY));
+			continue;
+		}
+
+		state ClusterConnectionString clusterConnectionString(currentKey.get().toString());
+		if (tr->getDatabase()->getConnectionRecord() &&
+		    clusterConnectionString.clusterKeyName().toString() !=
+		        tr->getDatabase()->getConnectionRecord()->getConnectionString().clusterKeyName()) {
+			// Someone changed the "name" of the database??
+			++retryTimes;
+			wait(delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY));
+			continue;
+		}
+
+		return clusterConnectionString;
+	}
+}
+
+} // namespace
+
+ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
+                                                               ClusterConnectionString* conn,
+                                                               std::string newName) {
+
+	state Optional<ClusterConnectionString> clusterConnectionStringOptional =
+	    wait(getClusterConnectionStringFromStorageServer(tr));
+
+	if (!clusterConnectionStringOptional.present()) {
+		return CoordinatorsResult::BAD_DATABASE_STATE;
+	}
+
+	// The cluster connection string stored in the storage server
+	state ClusterConnectionString old = clusterConnectionStringOptional.get();

 	if (conn->hostnames.size() + conn->coords.size() == 0) {
 		conn->hostnames = old.hostnames;
@ -896,28 +933,26 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan

 	loop {
 		try {
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			Optional<Value> currentKey = wait(tr.get(coordinatorsKey));
+			state Optional<ClusterConnectionString> clusterConnectionStringOptional =
+			    wait(getClusterConnectionStringFromStorageServer(&tr));

-			if (!currentKey.present())
-				return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
+			if (!clusterConnectionStringOptional.present()) {
+				return CoordinatorsResult::BAD_DATABASE_STATE;
+			}

-			state ClusterConnectionString old(currentKey.get().toString());
-			if (cx->getConnectionRecord() &&
-			    old.clusterKeyName().toString() != cx->getConnectionRecord()->getConnectionString().clusterKeyName())
-				return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
+			// The cluster connection string stored in the storage server
+			state ClusterConnectionString oldClusterConnectionString = clusterConnectionStringOptional.get();
+			state Key oldClusterKeyName = oldClusterConnectionString.clusterKeyName();

-			state std::vector<NetworkAddress> oldCoordinators = wait(old.tryResolveHostnames());
+			state std::vector<NetworkAddress> oldCoordinators = wait(oldClusterConnectionString.tryResolveHostnames());
 			state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 			if (!desiredCoordinators.size()) {
-				std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
-				    &tr,
-				    oldCoordinators,
-				    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
-				    result));
+				std::vector<NetworkAddress> _desiredCoordinators = wait(
+				    change->getDesiredCoordinators(&tr,
+				                                   oldCoordinators,
+				                                   Reference<ClusterConnectionMemoryRecord>(
+				                                       new ClusterConnectionMemoryRecord(oldClusterConnectionString)),
+				                                   result));
 				desiredCoordinators = _desiredCoordinators;
 			}

@ -937,13 +972,14 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan

 			std::string newName = change->getDesiredClusterKeyName();
 			if (newName.empty())
-				newName = old.clusterKeyName().toString();
+				newName = oldClusterKeyName.toString();

-			if (oldCoordinators == desiredCoordinators && old.clusterKeyName() == newName)
+			if (oldCoordinators == desiredCoordinators && oldClusterKeyName == newName)
 				return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES;

-			state ClusterConnectionString conn(
+			state ClusterConnectionString newClusterConnectionString(
 			    desiredCoordinators, StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
+			state Key newClusterKeyName = newClusterConnectionString.clusterKeyName();

 			if (g_network->isSimulated()) {
 				for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) {
@ -958,19 +994,22 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				}
 			}

-			TraceEvent("AttemptingQuorumChange").detail("FromCS", old.toString()).detail("ToCS", conn.toString());
-			TEST(old.clusterKeyName() != conn.clusterKeyName()); // Quorum change with new name
-			TEST(old.clusterKeyName() == conn.clusterKeyName()); // Quorum change with unchanged name
+			TraceEvent("AttemptingQuorumChange")
+			    .detail("FromCS", oldClusterConnectionString.toString())
+			    .detail("ToCS", newClusterConnectionString.toString());
+			TEST(oldClusterKeyName != newClusterKeyName); // Quorum change with new name
+			TEST(oldClusterKeyName == newClusterKeyName); // Quorum change with unchanged name

 			state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-			state ClientCoordinators coord(
-			    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(conn)));
+			state ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(
+			    new ClusterConnectionMemoryRecord(newClusterConnectionString)));
 			// check if allowed to modify the cluster descriptor
 			if (!change->getDesiredClusterKeyName().empty()) {
 				CheckDescriptorMutableReply mutabilityReply =
 				    wait(coord.clientLeaderServers[0].checkDescriptorMutable.getReply(CheckDescriptorMutableRequest()));
-				if (!mutabilityReply.isMutable)
+				if (!mutabilityReply.isMutable) {
 					return CoordinatorsResult::BAD_DATABASE_STATE;
+				}
 			}
 			leaderServers.reserve(coord.clientLeaderServers.size());
 			for (int i = 0; i < coord.clientLeaderServers.size(); i++)
@ -982,7 +1021,7 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
 			}

-			tr.set(coordinatorsKey, conn.toString());
+			tr.set(coordinatorsKey, newClusterConnectionString.toString());

 			wait(tr.commit());
 			ASSERT(false); // commit should fail, but the value has changed
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@ -485,6 +485,14 @@ ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
 	                                             TaskPriority::Coordination);
 }

+std::string ClientLeaderRegInterface::getAddressString() const {
+	if (hostname.present()) {
+		return hostname.get().toString();
+	} else {
+		return getLeader.getEndpoint().getPrimaryAddress().toString();
+	}
+}
+
 // Nominee is the worker among all workers that are considered as leader by one coordinator
 // This function contacts a coordinator coord to ask who is its nominee.
 ACTOR Future<Void> monitorNominee(Key key,
@ -510,9 +518,7 @@ ACTOR Future<Void> monitorNominee(Key key,

 		TraceEvent("GetLeaderReply")
 		    .suppressFor(1.0)
-		    .detail("Coordinator",
-		            coord.hostname.present() ? coord.hostname.get().toString()
-		                                     : coord.getLeader.getEndpoint().getPrimaryAddress().toString())
+		    .detail("Coordinator", coord.getAddressString())
 		    .detail("Nominee", li.present() ? li.get().changeID : UID())
 		    .detail("ClusterKey", key.printable());

@ -552,7 +558,7 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 	          maskedNominees.end(),
 	          [](const std::pair<UID, int>& l, const std::pair<UID, int>& r) { return l.first < r.first; });

-	int bestCount = 0;
+	int bestCount = 1;
 	int bestIdx = 0;
 	int currentIdx = 0;
 	int curCount = 1;
@ -560,17 +566,13 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 		if (maskedNominees[currentIdx].first == maskedNominees[i].first) {
 			curCount++;
 		} else {
-			if (curCount > bestCount) {
-				bestIdx = currentIdx;
-				bestCount = curCount;
-			}
 			currentIdx = i;
 			curCount = 1;
 		}
-	}
-	if (curCount > bestCount) {
-		bestIdx = currentIdx;
-		bestCount = curCount;
+		if (curCount > bestCount) {
+			bestIdx = currentIdx;
+			bestCount = curCount;
+		}
 	}

 	bool majority = bestCount >= nominees.size() / 2 + 1;
@ -585,6 +587,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 	state AsyncTrigger nomineeChange;
 	state std::vector<Optional<LeaderInfo>> nominees;
 	state Future<Void> allActors;
+	state Optional<std::pair<LeaderInfo, bool>> leader;

 	nominees.resize(coordinators.clientLeaderServers.size());

@ -598,7 +601,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 	allActors = waitForAll(actors);

 	loop {
-		Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
+		leader = getLeader(nominees);
 		TraceEvent("MonitorLeaderChange")
 		    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
 		if (leader.present()) {
@ -619,7 +622,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 					    .detail("CurrentConnectionString",
 					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
+				wait(connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString()));
 				info.intermediateConnRecord = connRecord;
 			}

@ -875,6 +878,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 	state std::vector<UID> lastGrvProxyUIDs;
 	state std::vector<GrvProxyInterface> lastGrvProxies;
 	state std::vector<ClientLeaderRegInterface> clientLeaderServers;
+	state bool allConnectionsFailed = false;

 	clientLeaderServers.reserve(coordinatorsSize);
 	for (const auto& h : cs.hostnames) {
@ -900,7 +904,22 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		state ClusterConnectionString storedConnectionString;
 		if (connRecord) {
 			bool upToDate = wait(connRecord->upToDate(storedConnectionString));
-			if (!upToDate) {
+			if (upToDate) {
+				incorrectTime = Optional<double>();
+			} else if (allConnectionsFailed) {
+				// Failed to connect to all coordinators from the current connection string,
+				// so it is not possible to get any new updates from the cluster. It can be that
+				// all the coordinators have changed, but the client missed that, because it had
+				// an incompatible protocol version. Since the cluster file is different,
+				// it may have been updated by other clients.
+				TraceEvent("UpdatingConnectionStringFromFile")
+				    .detail("ClusterFile", connRecord->toString())
+				    .detail("StoredConnectionString", storedConnectionString.toString())
+				    .detail("CurrentConnectionString", connRecord->getConnectionString().toString());
+				wait(connRecord->setAndPersistConnectionString(storedConnectionString));
+				info.intermediateConnRecord = connRecord;
+				return info;
+			} else {
 				req.issues.push_back_deep(req.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
 				std::string connectionString = connRecord->getConnectionString().toString();
 				if (!incorrectTime.present()) {
@ -913,8 +932,6 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 				    .detail("ClusterFile", connRecord->toString())
 				    .detail("StoredConnectionString", storedConnectionString.toString())
 				    .detail("CurrentConnectionString", connectionString);
-			} else {
-				incorrectTime = Optional<double>();
 			}
 		} else {
 			incorrectTime = Optional<double>();
@ -957,7 +974,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 					    .detail("CurrentConnectionString",
 					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
+				wait(connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString()));
 				info.intermediateConnRecord = connRecord;
 			}

@ -968,11 +985,16 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
 			clientInfo->setUnconditional(ni);
 			successIndex = index;
+			allConnectionsFailed = false;
 		} else {
 			TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller
 			TEST(rep.getError().code() == error_code_lookup_failed); // Coordinator hostname resolving failure
+			TraceEvent("MonitorProxiesConnectFailed")
+			    .detail("Error", rep.getError().name())
+			    .detail("Coordinator", clientLeaderServer.getAddressString());
 			index = (index + 1) % coordinatorsSize;
 			if (index == successIndex) {
+				allConnectionsFailed = true;
 				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
 			}
 		}
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@ -385,7 +385,7 @@ void DLTransaction::reset() {
 	api->transactionReset(tr);
 }

-VersionVector DLTransaction::getVersionVector() {
+ThreadFuture<VersionVector> DLTransaction::getVersionVector() {
 	return VersionVector(); // not implemented
 }

@ -398,6 +398,38 @@ Reference<ITransaction> DLTenant::createTransaction() {
 	return Reference<ITransaction>(new DLTransaction(api, tr));
 }

+ThreadFuture<Key> DLTenant::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
+	if (!api->tenantPurgeBlobGranules) {
+		return unsupported_operation();
+	}
+	FdbCApi::FDBFuture* f = api->tenantPurgeBlobGranules(tenant,
+	                                                     keyRange.begin.begin(),
+	                                                     keyRange.begin.size(),
+	                                                     keyRange.end.begin(),
+	                                                     keyRange.end.size(),
+	                                                     purgeVersion,
+	                                                     force);
+
+	return toThreadFuture<Key>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		const uint8_t* key;
+		int keyLength;
+		FdbCApi::fdb_error_t error = api->futureGetKey(f, &key, &keyLength);
+		ASSERT(!error);
+
+		// The memory for this is stored in the FDBFuture and is released when the future gets destroyed
+		return Key(KeyRef(key, keyLength), Arena());
+	});
+}
+
+ThreadFuture<Void> DLTenant::waitPurgeGranulesComplete(const KeyRef& purgeKey) {
+	if (!api->tenantWaitPurgeGranulesComplete) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture* f = api->tenantWaitPurgeGranulesComplete(tenant, purgeKey.begin(), purgeKey.size());
+	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
+}
+
 // DLDatabase
 DLDatabase::DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture) : api(api), db(nullptr) {
 	addref();
@ -520,16 +552,16 @@ ThreadFuture<ProtocolVersion> DLDatabase::getServerProtocol(Optional<ProtocolVer
 }

 ThreadFuture<Key> DLDatabase::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
-	if (!api->purgeBlobGranules) {
+	if (!api->databasePurgeBlobGranules) {
 		return unsupported_operation();
 	}
-	FdbCApi::FDBFuture* f = api->purgeBlobGranules(db,
-	                                               keyRange.begin.begin(),
-	                                               keyRange.begin.size(),
-	                                               keyRange.end.begin(),
-	                                               keyRange.end.size(),
-	                                               purgeVersion,
-	                                               force);
+	FdbCApi::FDBFuture* f = api->databasePurgeBlobGranules(db,
+	                                                       keyRange.begin.begin(),
+	                                                       keyRange.begin.size(),
+	                                                       keyRange.end.begin(),
+	                                                       keyRange.end.size(),
+	                                                       purgeVersion,
+	                                                       force);

 	return toThreadFuture<Key>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
 		const uint8_t* key;
@ -543,11 +575,11 @@ ThreadFuture<Key> DLDatabase::purgeBlobGranules(const KeyRangeRef& keyRange, Ver
 }

 ThreadFuture<Void> DLDatabase::waitPurgeGranulesComplete(const KeyRef& purgeKey) {
-	if (!api->waitPurgeGranulesComplete) {
+	if (!api->databaseWaitPurgeGranulesComplete) {
 		return unsupported_operation();
 	}

-	FdbCApi::FDBFuture* f = api->waitPurgeGranulesComplete(db, purgeKey.begin(), purgeKey.size());
+	FdbCApi::FDBFuture* f = api->databaseWaitPurgeGranulesComplete(db, purgeKey.begin(), purgeKey.size());
 	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
 }

@ -629,11 +661,9 @@ void DLApi::init() {
 	                   headerVersion >= 700);
 	loadClientFunction(
 	    &api->databaseCreateSnapshot, lib, fdbCPath, "fdb_database_create_snapshot", headerVersion >= 700);
-
 	loadClientFunction(
-	    &api->purgeBlobGranules, lib, fdbCPath, "fdb_database_purge_blob_granules", headerVersion >= 710);
-
-	loadClientFunction(&api->waitPurgeGranulesComplete,
+	    &api->databasePurgeBlobGranules, lib, fdbCPath, "fdb_database_purge_blob_granules", headerVersion >= 710);
+	loadClientFunction(&api->databaseWaitPurgeGranulesComplete,
 	                   lib,
 	                   fdbCPath,
 	                   "fdb_database_wait_purge_granules_complete",
@ -641,6 +671,13 @@ void DLApi::init() {

 	loadClientFunction(
 	    &api->tenantCreateTransaction, lib, fdbCPath, "fdb_tenant_create_transaction", headerVersion >= 710);
+	loadClientFunction(
+	    &api->tenantPurgeBlobGranules, lib, fdbCPath, "fdb_tenant_purge_blob_granules", headerVersion >= 720);
+	loadClientFunction(&api->tenantWaitPurgeGranulesComplete,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_tenant_wait_purge_granules_complete",
+	                   headerVersion >= 720);
 	loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710);

 	loadClientFunction(&api->transactionSetOption, lib, fdbCPath, "fdb_transaction_set_option", headerVersion >= 0);
@ -1126,7 +1163,7 @@ Version MultiVersionTransaction::getCommittedVersion() {
 	return invalidVersion;
 }

-VersionVector MultiVersionTransaction::getVersionVector() {
+ThreadFuture<VersionVector> MultiVersionTransaction::getVersionVector() {
 	auto tr = getTransaction();
 	if (tr.transaction) {
 		return tr.transaction->getVersionVector();
@ -1135,7 +1172,7 @@ VersionVector MultiVersionTransaction::getVersionVector() {
 	return VersionVector();
 }

-SpanContext MultiVersionTransaction::getSpanContext() {
+ThreadFuture<SpanContext> MultiVersionTransaction::getSpanContext() {
 	auto tr = getTransaction();
 	if (tr.transaction) {
 		return tr.transaction->getSpanContext();
@ -1330,7 +1367,7 @@ bool MultiVersionTransaction::isValid() {
 }

 // MultiVersionTenant
-MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, StringRef tenantName)
+MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
  : tenantState(makeReference<TenantState>(db, tenantName)) {}

 MultiVersionTenant::~MultiVersionTenant() {
@ -1343,7 +1380,17 @@ Reference<ITransaction> MultiVersionTenant::createTransaction() {
 	                                                           tenantState->db->dbState->transactionDefaultOptions));
 }

-MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, StringRef tenantName)
+ThreadFuture<Key> MultiVersionTenant::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
+	auto f = tenantState->db ? tenantState->db->purgeBlobGranules(keyRange, purgeVersion, force)
+	                         : ThreadFuture<Key>(Never());
+	return abortableFuture(f, tenantState->db->dbState->dbVar->get().onChange);
+}
+ThreadFuture<Void> MultiVersionTenant::waitPurgeGranulesComplete(const KeyRef& purgeKey) {
+	auto f = tenantState->db ? tenantState->db->waitPurgeGranulesComplete(purgeKey) : ThreadFuture<Void>(Never());
+	return abortableFuture(f, tenantState->db->dbState->dbVar->get().onChange);
+}
+
+MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
  : tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db),
    closed(false) {
 	updateTenant();
@ -1441,8 +1488,7 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
 		});

 		Reference<DatabaseState> dbStateRef = dbState;
-		onMainThreadVoid([dbStateRef]() { dbStateRef->protocolVersionMonitor = dbStateRef->monitorProtocolVersion(); },
-		                 nullptr);
+		onMainThreadVoid([dbStateRef]() { dbStateRef->protocolVersionMonitor = dbStateRef->monitorProtocolVersion(); });
 	}
 }

@ -1616,7 +1662,7 @@ ThreadFuture<Void> MultiVersionDatabase::DatabaseState::monitorProtocolVersion()

 		ProtocolVersion clusterVersion =
 		    !cv.isError() ? cv.get() : self->dbProtocolVersion.orDefault(currentProtocolVersion);
-		onMainThreadVoid([self, clusterVersion]() { self->protocolVersionChanged(clusterVersion); }, nullptr);
+		onMainThreadVoid([self, clusterVersion]() { self->protocolVersionChanged(clusterVersion); });
 		return ErrorOr<Void>(Void());
 	});
 }
@ -1681,10 +1727,10 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
 				dbReady = mapThreadFuture<Void, Void>(
 				    newDb.castTo<DLDatabase>()->onReady(), [self, newDb, client](ErrorOr<Void> ready) {
 					    if (!ready.isError()) {
-						    onMainThreadVoid([self, newDb, client]() { self->updateDatabase(newDb, client); }, nullptr);
+						    onMainThreadVoid([self, newDb, client]() { self->updateDatabase(newDb, client); });
 					    } else {
-						    onMainThreadVoid([self, client]() { self->updateDatabase(Reference<IDatabase>(), client); },
-						                     nullptr);
+						    onMainThreadVoid(
+						        [self, client]() { self->updateDatabase(Reference<IDatabase>(), client); });
 					    }

 					    return ready;
@ -1794,19 +1840,17 @@ void MultiVersionDatabase::DatabaseState::startLegacyVersionMonitors() {
 // Cleans up state for the legacy version monitors to break reference cycles
 void MultiVersionDatabase::DatabaseState::close() {
 	Reference<DatabaseState> self = Reference<DatabaseState>::addRef(this);
-	onMainThreadVoid(
-	    [self]() {
-		    self->closed = true;
-		    if (self->protocolVersionMonitor.isValid()) {
-			    self->protocolVersionMonitor.cancel();
-		    }
-		    for (auto monitor : self->legacyVersionMonitors) {
-			    monitor->close();
-		    }
+	onMainThreadVoid([self]() {
+		self->closed = true;
+		if (self->protocolVersionMonitor.isValid()) {
+			self->protocolVersionMonitor.cancel();
+		}
+		for (auto monitor : self->legacyVersionMonitors) {
+			monitor->close();
+		}

-		    self->legacyVersionMonitors.clear();
-	    },
-	    nullptr);
+		self->legacyVersionMonitors.clear();
+	});
 }

 // Starts the connection monitor by creating a database object at an old version.
@ -1826,22 +1870,20 @@ void MultiVersionDatabase::LegacyVersionMonitor::startConnectionMonitor(
 		Reference<LegacyVersionMonitor> self = Reference<LegacyVersionMonitor>::addRef(this);
 		versionMonitor =
 		    mapThreadFuture<Void, Void>(db.castTo<DLDatabase>()->onReady(), [self, dbState](ErrorOr<Void> ready) {
-			    onMainThreadVoid(
-			        [self, ready, dbState]() {
-				        if (ready.isError()) {
-					        if (ready.getError().code() != error_code_operation_cancelled) {
-						        TraceEvent(SevError, "FailedToOpenDatabaseOnClient")
-						            .error(ready.getError())
-						            .detail("LibPath", self->client->libPath);
+			    onMainThreadVoid([self, ready, dbState]() {
+				    if (ready.isError()) {
+					    if (ready.getError().code() != error_code_operation_cancelled) {
+						    TraceEvent(SevError, "FailedToOpenDatabaseOnClient")
+						        .error(ready.getError())
+						        .detail("LibPath", self->client->libPath);

-						        self->client->failed = true;
-						        MultiVersionApi::api->updateSupportedVersions();
-					        }
-				        } else {
-					        self->runGrvProbe(dbState);
-				        }
-			        },
-			        nullptr);
+						    self->client->failed = true;
+						    MultiVersionApi::api->updateSupportedVersions();
+					    }
+				    } else {
+					    self->runGrvProbe(dbState);
+				    }
+			    });

 			    return ready;
 		    });
@ -1856,12 +1898,10 @@ void MultiVersionDatabase::LegacyVersionMonitor::runGrvProbe(Reference<MultiVers
 	versionMonitor = mapThreadFuture<Version, Void>(tr->getReadVersion(), [self, dbState](ErrorOr<Version> v) {
 		// If the version attempt returns an error, we regard that as a connection (except operation_cancelled)
 		if (!v.isError() || v.getError().code() != error_code_operation_cancelled) {
-			onMainThreadVoid(
-			    [self, dbState]() {
-				    self->monitorRunning = false;
-				    dbState->protocolVersionChanged(self->client->protocolVersion);
-			    },
-			    nullptr);
+			onMainThreadVoid([self, dbState]() {
+				self->monitorRunning = false;
+				dbState->protocolVersionChanged(self->client->protocolVersion);
+			});
 		}

 		return v.map<Void>([](Version v) { return Void(); });
@ -2111,11 +2151,9 @@ void MultiVersionApi::setSupportedClientVersions(Standalone<StringRef> versions)

 	// This option must be set on the main thread because it modifies structures that can be used concurrently by the
 	// main thread
-	onMainThreadVoid(
-	    [this, versions]() {
-		    localClient->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions);
-	    },
-	    nullptr);
+	onMainThreadVoid([this, versions]() {
+		localClient->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions);
+	});

 	if (!bypassMultiClientApi) {
 		runOnExternalClientsAllThreads([versions](Reference<ClientInfo> client) {
@ -2883,7 +2921,7 @@ THREAD_FUNC runSingleAssignmentVarTest(void* arg) {
 			checkUndestroyed.blockUntilReady();
 		}

-		onMainThreadVoid([done]() { *done = true; }, nullptr);
+		onMainThreadVoid([done]() { *done = true; });
 	} catch (Error& e) {
 		printf("Caught error in test: %s\n", e.name());
 		*done = true;
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@ -31,7 +31,7 @@
 #include <vector>

 #include "boost/algorithm/string.hpp"
-#include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fmt/format.h"

 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
@ -63,7 +63,7 @@
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TransactionLineage.h"
 #include "fdbclient/versions.h"
-#include "fdbclient/WellKnownEndpoints.h"
+#include "fdbrpc/WellKnownEndpoints.h"
 #include "fdbrpc/LoadBalance.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
@ -82,7 +82,7 @@
 #include "flow/Platform.h"
 #include "flow/SystemMonitor.h"
 #include "flow/TLSConfig.actor.h"
-#include "flow/Tracing.h"
+#include "fdbclient/Tracing.h"
 #include "flow/UnitTest.h"
 #include "flow/network.h"
 #include "flow/serialize.h"
@ -1036,6 +1036,7 @@ ACTOR Future<Void> monitorCacheList(DatabaseContext* self) {
 	state Transaction tr;
 	state std::map<UID, StorageServerInterface> cacheServerMap;
 	state Future<Void> updateRanges = updateCachedRanges(self, &cacheServerMap);
+	state Backoff backoff;
 	// if no caches are configured, we don't want to run this actor at all
 	// so we just wait for the first trigger from a storage server
 	wait(self->updateCache.onTrigger());
@ -1073,8 +1074,10 @@ ACTOR Future<Void> monitorCacheList(DatabaseContext* self) {
 				}
 				cacheServerMap = std::move(allCacheServers);
 				wait(delay(5.0));
+				backoff = Backoff();
 			} catch (Error& e) {
 				wait(tr.onError(e));
+				wait(backoff.onError());
 			}
 		}
 	} catch (Error& e) {
@ -1154,6 +1157,7 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 	state Transaction tr;
 	state double grvDelay = 0.001;
+	state Backoff backoff;
 	try {
 		loop {
 			if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF)
@ -1181,9 +1185,11 @@ ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
 					    .detail("GrvDelay", grvDelay)
 					    .detail("CachedReadVersion", cx->getCachedReadVersion())
 					    .detail("CachedTime", cx->getLastGrvTime());
+					backoff = Backoff();
 				} catch (Error& e) {
 					TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").errorUnsuppressed(e);
 					wait(tr.onError(e));
+					wait(backoff.onError());
 				}
 			} else {
 				wait(
@ -7016,7 +7022,7 @@ ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
 		} else {
 			state NetworkAddress coordinatorAddress;
 			if (coordinator->get().get().hostname.present()) {
-				Hostname h = coordinator->get().get().hostname.get();
+				state Hostname h = coordinator->get().get().hostname.get();
 				wait(store(coordinatorAddress, h.resolveWithRetry()));
 			} else {
 				coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
@ -7515,14 +7521,42 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Trans
 	// FIXME: use streaming range read
 	state KeyRange currentRange = keyRange;
 	state Standalone<VectorRef<KeyRangeRef>> results;
+	state Optional<Key> tenantPrefix;
 	if (BG_REQUEST_DEBUG) {
 		fmt::print("Getting Blob Granules for [{0} - {1})\n", keyRange.begin.printable(), keyRange.end.printable());
 	}
-	self->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	// FIXME: limit to tenant range if set
+
+	if (self->getTenant().present()) {
+		// have to bypass tenant to read system key space, and add tenant prefix to part of mapping
+		TenantMapEntry tenantEntry = wait(blobGranuleGetTenantEntry(self, currentRange.begin));
+		tenantPrefix = tenantEntry.prefix;
+	} else {
+		self->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+	}
 	loop {
-		state RangeResult blobGranuleMapping = wait(
-		    krmGetRanges(self, blobGranuleMappingKeys.begin, currentRange, 1000, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+		state RangeResult blobGranuleMapping;
+		if (tenantPrefix.present()) {
+			state Standalone<StringRef> mappingPrefix = tenantPrefix.get().withPrefix(blobGranuleMappingKeys.begin);
+
+			// basically krmGetRange, but enable it to not use tenant without RAW_ACCESS by doing manual getRange with
+			// UseTenant::False
+			GetRangeLimits limits(1000);
+			limits.minRows = 2;
+			RangeResult rawMapping = wait(getRange(self->trState,
+			                                       self->getReadVersion(),
+			                                       lastLessOrEqual(keyRange.begin.withPrefix(mappingPrefix)),
+			                                       firstGreaterThan(keyRange.end.withPrefix(mappingPrefix)),
+			                                       limits,
+			                                       Reverse::False,
+			                                       UseTenant::False));
+			// strip off mapping prefix
+			blobGranuleMapping = krmDecodeRanges(mappingPrefix, currentRange, rawMapping);
+		} else {
+			wait(store(
+			    blobGranuleMapping,
+			    krmGetRanges(
+			        self, blobGranuleMappingKeys.begin, currentRange, 1000, GetRangeLimits::BYTE_LIMIT_UNLIMITED)));
+		}

 		for (int i = 0; i < blobGranuleMapping.size() - 1; i++) {
 			if (blobGranuleMapping[i].value.size()) {
@ -9448,24 +9482,33 @@ Reference<DatabaseContext::TransactionT> DatabaseContext::createTransaction() {
 	return makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(this)));
 }

-// FIXME: handle tenants?
 ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
                                         KeyRange range,
                                         Version purgeVersion,
+                                         Optional<TenantName> tenant,
                                         bool force) {
 	state Database cx(db);
 	state Transaction tr(cx);
 	state Key purgeKey;
+	state KeyRange purgeRange = range;
+	state bool loadedTenantPrefix = false;

 	// FIXME: implement force
 	if (!force) {
 		throw unsupported_operation();
 	}
+
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);

+			if (tenant.present() && !loadedTenantPrefix) {
+				TenantMapEntry tenantEntry = wait(blobGranuleGetTenantEntry(&tr, range.begin));
+				loadedTenantPrefix = true;
+				purgeRange = purgeRange.withPrefix(tenantEntry.prefix);
+			}
+
 			Value purgeValue = blobGranulePurgeValueFor(purgeVersion, range, force);
 			tr.atomicOp(
 			    addVersionStampAtEnd(blobGranulePurgeKeys.begin), purgeValue, MutationRef::SetVersionstampedKey);
@ -9496,8 +9539,11 @@ ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
 	return purgeKey;
 }

-Future<Key> DatabaseContext::purgeBlobGranules(KeyRange range, Version purgeVersion, bool force) {
-	return purgeBlobGranulesActor(Reference<DatabaseContext>::addRef(this), range, purgeVersion, force);
+Future<Key> DatabaseContext::purgeBlobGranules(KeyRange range,
+                                               Version purgeVersion,
+                                               Optional<TenantName> tenant,
+                                               bool force) {
+	return purgeBlobGranulesActor(Reference<DatabaseContext>::addRef(this), range, purgeVersion, tenant, force);
 }

 ACTOR Future<Void> waitPurgeGranulesCompleteActor(Reference<DatabaseContext> db, Key purgeKey) {
@ -9549,3 +9595,27 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
 int64_t getMaxClearKeySize(KeyRef const& key) {
 	return getMaxKeySize(key);
 }
+
+namespace NativeAPI {
+
+ACTOR Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(
+    Transaction* tr) {
+	state Future<std::vector<ProcessData>> workers = getWorkers(tr);
+	state Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+	wait(success(workers) && success(serverList));
+	ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
+
+	std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
+	for (int i = 0; i < workers.get().size(); i++)
+		id_data[workers.get()[i].locality.processId()] = workers.get()[i];
+
+	std::vector<std::pair<StorageServerInterface, ProcessClass>> results;
+	for (int i = 0; i < serverList.get().size(); i++) {
+		auto ssi = decodeServerListValue(serverList.get()[i].value);
+		results.emplace_back(ssi, id_data[ssi.locality.processId()].processClass);
+	}
+
+	return results;
+}
+
+} // namespace NativeAPI
--- a/fdbclient/RESTClient.actor.cpp
+++ b/fdbclient/RESTClient.actor.cpp
@ -18,11 +18,11 @@
 * limitations under the License.
 */

-#include "fdbrpc/RESTClient.h"
+#include "fdbclient/RESTClient.h"

-#include "fdbrpc/HTTP.h"
-#include "fdbrpc/IRateControl.h"
-#include "fdbrpc/RESTUtils.h"
+#include "fdbclient/HTTP.h"
+#include "flow/IRateControl.h"
+#include "fdbclient/RESTUtils.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
--- a/fdbclient/RESTUtils.actor.cpp
+++ b/fdbclient/RESTUtils.actor.cpp
@ -18,7 +18,7 @@
 * limitations under the License.
 */

-#include "fdbrpc/RESTUtils.h"
+#include "fdbclient/RESTUtils.h"

 #include "flow/flat_buffers.h"
 #include "flow/UnitTest.h"
@ -268,4 +268,4 @@ TEST_CASE("/RESTUtils/ValidURIWithParams") {
 	ASSERT_EQ(r.resource.compare("foo/bar"), 0);
 	ASSERT_EQ(r.reqParameters.compare("param1,param2"), 0);
 	return Void();
-}
+}
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@ -37,11 +37,13 @@
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string.hpp>
-#include "fdbrpc/IAsyncFile.h"
+#include "flow/IAsyncFile.h"
 #include "flow/Hostname.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/rapidxml/rapidxml.hpp"
+#ifdef BUILD_AWS_BACKUP
 #include "fdbclient/FDBAWSCredentialsProvider.h"
+#endif

 #include "flow/actorcompiler.h" // has to be last include

@ -169,6 +171,32 @@ std::string S3BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
 	return r;
 }

+std::string guessRegionFromDomain(std::string domain) {
+	static const std::vector<const char*> knownServices = { "s3.", "cos.", "oss-", "obs." };
+	boost::algorithm::to_lower(domain);
+
+	for (int i = 0; i < knownServices.size(); ++i) {
+		const char* service = knownServices[i];
+
+		std::size_t p = domain.find(service);
+
+		if (p == std::string::npos || (p >= 1 && domain[p - 1] != '.')) {
+			// eg. 127.0.0.1, example.com, s3-service.example.com, mys3.example.com
+			continue;
+		}
+
+		StringRef h(domain.c_str() + p);
+
+		if (!h.startsWith(LiteralStringRef("oss-"))) {
+			h.eat(service); // ignore s3 service
+		}
+
+		return h.eat(".").toString();
+	}
+
+	return "";
+}
+
 Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string& url,
                                                               const Optional<std::string>& proxy,
                                                               std::string* resourceFromURL,
@ -222,6 +250,8 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string

 		StringRef service = h.eat();

+		std::string region = guessRegionFromDomain(host.toString());
+
 		BlobKnobs knobs;
 		HTTP::Headers extraHeaders;
 		while (1) {
@ -251,6 +281,12 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 				continue;
 			}

+			// overwrite s3 region from parameter
+			if (name == LiteralStringRef("region")) {
+				region = value.toString();
+				continue;
+			}
+
 			// See if the parameter is a knob
 			// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is known
 			// to S3BlobStoreEndpoint. If it is, then we will set it to a good value or throw below, so the dummy set
@ -289,8 +325,13 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			creds = S3BlobStoreEndpoint::Credentials{ key.toString(), secret.toString(), securityToken.toString() };
 		}

+		if (region.empty() && CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
+			throw std::string(
+			    "Failed to get region from host or parameter in url, region is required for aws v4 signature");
+		}
+
 		return makeReference<S3BlobStoreEndpoint>(
-		    host.toString(), service.toString(), proxyHost, proxyPort, creds, knobs, extraHeaders);
+		    host.toString(), service.toString(), region, proxyHost, proxyPort, creds, knobs, extraHeaders);

 	} catch (std::string& err) {
 		if (error != nullptr)
@ -356,10 +397,27 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin
 	return r;
 }

+std::string constructResourcePath(Reference<S3BlobStoreEndpoint> b,
+                                  const std::string& bucket,
+                                  const std::string& object) {
+	std::string resource;
+
+	if (b->getHost().find(bucket + ".") != 0) {
+		resource += std::string("/") + bucket; // not virtual hosting mode
+	}
+
+	if (!object.empty()) {
+		resource += "/";
+		resource += object;
+	}
+
+	return resource;
+}
+
 ACTOR Future<bool> bucketExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
 	wait(b->requestRateRead->getAllowance(1));

-	std::string resource = std::string("/") + bucket;
+	std::string resource = constructResourcePath(b, bucket, "");
 	HTTP::Headers headers;

 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@ -373,7 +431,7 @@ Future<bool> S3BlobStoreEndpoint::bucketExists(std::string const& bucket) {
 ACTOR Future<bool> objectExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateRead->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;

 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@ -387,7 +445,7 @@ Future<bool> S3BlobStoreEndpoint::objectExists(std::string const& bucket, std::s
 ACTOR Future<Void> deleteObject_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateDelete->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;
 	// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered
 	// successful
@ -477,9 +535,24 @@ ACTOR Future<Void> createBucket_impl(Reference<S3BlobStoreEndpoint> b, std::stri

 	bool exists = wait(b->bucketExists(bucket));
 	if (!exists) {
-		std::string resource = std::string("/") + bucket;
+		std::string resource = constructResourcePath(b, bucket, "");
 		HTTP::Headers headers;
-		Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 }));
+
+		std::string region = b->getRegion();
+		if (region.empty()) {
+			Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 }));
+		} else {
+			UnsentPacketQueue packets;
+			StringRef body(format("<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+			                      "  <LocationConstraint>%s</LocationConstraint>"
+			                      "</CreateBucketConfiguration>",
+			                      region.c_str()));
+			PacketWriter pw(packets.getWriteBuffer(), nullptr, Unversioned());
+			pw.serializeBytes(body);
+
+			Reference<HTTP::Response> r =
+			    wait(b->doRequest("PUT", resource, headers, &packets, body.size(), { 200, 409 }));
+		}
 	}
 	return Void();
 }
@ -491,7 +564,7 @@ Future<Void> S3BlobStoreEndpoint::createBucket(std::string const& bucket) {
 ACTOR Future<int64_t> objectSize_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateRead->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;

 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@ -966,8 +1039,8 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
                                          int maxDepth,
                                          std::function<bool(std::string const&)> recurseFilter) {
 	// Request 1000 keys at a time, the maximum allowed
-	state std::string resource = "/";
-	resource.append(bucket);
+	state std::string resource = constructResourcePath(bstore, bucket, "");
+
 	resource.append("/?max-keys=1000");
 	if (prefix.present())
 		resource.append("&prefix=").append(prefix.get());
@ -1324,10 +1397,6 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
 		amzDate = date;
 		dateStamp = datestamp;
 	}
-	// Extract service and region
-	StringRef hostRef(host);
-	std::string service = hostRef.eat(".").toString();
-	std::string region = hostRef.eat(".").toString();

 	// ************* TASK 1: CREATE A CANONICAL REQUEST *************
 	// Create Create canonical URI--the part of the URI from domain to query string (use '/' if no path)
@ -1370,14 +1439,14 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,

 	// ************* TASK 2: CREATE THE STRING TO SIGN*************
 	std::string algorithm = "AWS4-HMAC-SHA256";
-	std::string credentialScope = dateStamp + "/" + region + "/" + service + "/" + "aws4_request";
+	std::string credentialScope = dateStamp + "/" + region + "/s3/" + "aws4_request";
 	std::string stringToSign =
 	    algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + sha256_hex(canonicalRequest);

 	// ************* TASK 3: CALCULATE THE SIGNATURE *************
 	// Create the signing key using the function defined above.
-	std::string signingKey = hmac_sha256(
-	    hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), service), "aws4_request");
+	std::string signingKey =
+	    hmac_sha256(hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), "s3"), "aws4_request");
 	// Sign the string_to_sign using the signing_key
 	std::string signature = hmac_sha256_hex(signingKey, stringToSign);
 	// ************* TASK 4: ADD SIGNING INFORMATION TO THE Header *************
@ -1445,7 +1514,7 @@ ACTOR Future<std::string> readEntireFile_impl(Reference<S3BlobStoreEndpoint> bst
                                              std::string object) {
 	wait(bstore->requestRateRead->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 404 }));
 	if (r->code == 404)
@ -1470,7 +1539,7 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<S3BlobStoreEndpoint>
 	wait(bstore->concurrentUploads.take());
 	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	// Send MD5 sum for content so blobstore can verify it
 	headers["Content-MD5"] = contentMD5;
@ -1540,7 +1609,7 @@ ACTOR Future<int> readObject_impl(Reference<S3BlobStoreEndpoint> bstore,
 		return 0;
 	wait(bstore->requestRateRead->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1);
 	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 206, 404 }));
@ -1567,7 +1636,8 @@ ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<S3BlobStore
                                                           std::string object) {
 	wait(bstore->requestRateWrite->getAllowance(1));

-	std::string resource = std::string("/") + bucket + "/" + object + "?uploads";
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += "?uploads";
 	HTTP::Headers headers;
 	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
 		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
@ -1609,8 +1679,8 @@ ACTOR Future<std::string> uploadPart_impl(Reference<S3BlobStoreEndpoint> bstore,
 	wait(bstore->concurrentUploads.take());
 	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);

-	std::string resource =
-	    format("/%s/%s?partNumber=%d&uploadId=%s", bucket.c_str(), object.c_str(), partNumber, uploadID.c_str());
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += format("?partNumber=%d&uploadId=%s", partNumber, uploadID.c_str());
 	HTTP::Headers headers;
 	// Send MD5 sum for content so blobstore can verify it
 	headers["Content-MD5"] = contentMD5;
@ -1662,7 +1732,8 @@ ACTOR Future<Void> finishMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bst
 		manifest += format("<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>\n", p.first, p.second.c_str());
 	manifest += "</CompleteMultipartUpload>";

-	std::string resource = format("/%s/%s?uploadId=%s", bucket.c_str(), object.c_str(), uploadID.c_str());
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += format("?uploadId=%s", uploadID.c_str());
 	HTTP::Headers headers;
 	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned());
 	pw.serializeBytes(manifest);
@ -1686,7 +1757,7 @@ TEST_CASE("/backup/s3/v4headers") {
 	S3BlobStoreEndpoint::Credentials creds{ "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "" }
 	// GET without query parameters
 	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
 		std::string verb("GET");
 		std::string resource("/test.txt");
 		HTTP::Headers headers;
@ -1701,7 +1772,7 @@ TEST_CASE("/backup/s3/v4headers") {

 	// GET with query parameters
 	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
 		std::string verb("GET");
 		std::string resource("/test/examplebucket?Action=DescribeRegions&Version=2013-10-15");
 		HTTP::Headers headers;
@ -1716,7 +1787,7 @@ TEST_CASE("/backup/s3/v4headers") {

 	// POST
 	{
-		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "443", "us-west-2", "proxy", "port", creds);
 		std::string verb("POST");
 		std::string resource("/simple.json");
 		HTTP::Headers headers;
@ -1733,4 +1804,4 @@ TEST_CASE("/backup/s3/v4headers") {
 	}

 	return Void();
-}
+}
--- a/Show More
+++ b/Show More